@temporalio/core-bridge 0.16.4 → 0.18.0

package/sdk-core/client/src/metrics.rs

@@ -0,0 +1,89 @@
+use opentelemetry::{
+    metrics::{Counter, Meter, ValueRecorder},
+    KeyValue,
+};
+use std::{sync::Arc, time::Duration};
+
+/// Used to track context associated with metrics, and record/update them
+///
+/// Possible improvement: make generic over some type tag so that methods are only exposed if the
+/// appropriate k/vs have already been set.
+#[derive(Clone, Debug)]
+pub(crate) struct MetricsContext {
+    kvs: Arc<Vec<KeyValue>>,
+    poll_is_long: bool,
+
+    svc_request: Counter<u64>,
+    svc_request_failed: Counter<u64>,
+    long_svc_request: Counter<u64>,
+    long_svc_request_failed: Counter<u64>,
+
+    svc_request_latency: ValueRecorder<u64>,
+    long_svc_request_latency: ValueRecorder<u64>,
+}
+
+impl MetricsContext {
+    fn new(kvs: Vec<KeyValue>, meter: &Meter) -> Self {
+        Self {
+            kvs: Arc::new(kvs),
+            poll_is_long: false,
+            svc_request: meter.u64_counter("request").init(),
+            svc_request_failed: meter.u64_counter("request_failure").init(),
+            long_svc_request: meter.u64_counter("long_request").init(),
+            long_svc_request_failed: meter.u64_counter("long_request_failure").init(),
+            svc_request_latency: meter.u64_value_recorder("request_latency").init(),
+            long_svc_request_latency: meter.u64_value_recorder("long_request_latency").init(),
+        }
+    }
+
+    pub(crate) fn top_level(namespace: String, meter: &Meter) -> Self {
+        Self::new(vec![KeyValue::new(KEY_NAMESPACE, namespace)], meter)
+    }
+
+    /// Extend an existing metrics context with new attributes
+    pub(crate) fn with_new_attrs(&self, new_kvs: impl IntoIterator<Item = KeyValue>) -> Self {
+        let mut r = self.clone();
+        Arc::make_mut(&mut r.kvs).extend(new_kvs);
+        r
+    }
+
+    pub(crate) fn set_is_long_poll(&mut self) {
+        self.poll_is_long = true;
+    }
+
+    /// A request to the temporal service was made
+    pub(crate) fn svc_request(&self) {
+        if self.poll_is_long {
+            self.long_svc_request.add(1, &self.kvs);
+        } else {
+            self.svc_request.add(1, &self.kvs);
+        }
+    }
+
+    /// A request to the temporal service failed
+    pub(crate) fn svc_request_failed(&self) {
+        if self.poll_is_long {
+            self.long_svc_request_failed.add(1, &self.kvs);
+        } else {
+            self.svc_request_failed.add(1, &self.kvs);
+        }
+    }
+
+    /// Record service request latency
+    pub(crate) fn record_svc_req_latency(&self, dur: Duration) {
+        if self.poll_is_long {
+            self.long_svc_request_latency
+                .record(dur.as_millis() as u64, &self.kvs);
+        } else {
+            self.svc_request_latency
+                .record(dur.as_millis() as u64, &self.kvs);
+        }
+    }
+}
+
+const KEY_NAMESPACE: &str = "namespace";
+const KEY_SVC_METHOD: &str = "operation";
+
+pub(crate) fn svc_operation(op: String) -> KeyValue {
+    KeyValue::new(KEY_SVC_METHOD, op)
+}

package/sdk-core/client/src/mocks.rs

@@ -0,0 +1,167 @@
+//! Helpers for mocking
+
+use super::*;
+use crate::{MockServerGatewayApis, ServerGatewayOptions, ServerGatewayOptionsBuilder};
+use std::str::FromStr;
+use url::Url;
+
+/// Create a mock client primed with basic necessary expectations
+pub fn mock_gateway() -> MockServerGatewayApis {
+    let mut mg = MockServerGatewayApis::new();
+    mg.expect_get_options().return_const(fake_sg_opts());
+    mg
+}
+
+/// Create a mock manual client primed with basic necessary expectations
+pub fn mock_manual_gateway() -> MockManualGateway {
+    let mut mg = MockManualGateway::new();
+    mg.expect_get_options().return_const(fake_sg_opts());
+    mg
+}
+
+/// Returns some totally fake client options for use with mock clients
+pub fn fake_sg_opts() -> ServerGatewayOptions {
+    ServerGatewayOptionsBuilder::default()
+        .target_url(Url::from_str("https://fake").unwrap())
+        .namespace("test_namespace".to_string())
+        .client_name("fake_client".to_string())
+        .client_version("fake_version".to_string())
+        .worker_binary_id("fake_binid".to_string())
+        .build()
+        .unwrap()
+}
+
+// Need a version of the mock that can return futures so we can return potentially pending
+// results. This is really annoying b/c of the async trait stuff. Need
+// https://github.com/asomers/mockall/issues/189 to be fixed for it to go away.
+mockall::mock! {
+    pub ManualGateway {}
+    impl ServerGatewayApis for ManualGateway {
+        fn start_workflow<'a, 'b>(
+            &self,
+            input: Vec<Payload>,
+            task_queue: String,
+            workflow_id: String,
+            workflow_type: String,
+            task_timeout: Option<Duration>,
+        ) -> impl Future<Output = Result<StartWorkflowExecutionResponse>> + Send + 'b
+            where 'a: 'b, Self: 'b;
+
+        fn poll_workflow_task<'a, 'b>(&'a self, task_queue: String, is_sticky: bool)
+            -> impl Future<Output = Result<PollWorkflowTaskQueueResponse>> + Send + 'b
+            where 'a: 'b, Self: 'b;
+
+        fn poll_activity_task<'a, 'b>(&self, task_queue: String)
+            -> impl Future<Output = Result<PollActivityTaskQueueResponse>> + Send + 'b
+            where 'a: 'b, Self: 'b;
+
+        fn reset_sticky_task_queue<'a, 'b>(
+            &self,
+            workflow_id: String,
+            run_id: String,
+        ) -> impl Future<Output = Result<ResetStickyTaskQueueResponse>> + Send + 'b
+            where 'a: 'b, Self: 'b;
+
+        fn complete_workflow_task<'a, 'b>(
+            &self,
+            request: WorkflowTaskCompletion,
+        ) -> impl Future<Output = Result<RespondWorkflowTaskCompletedResponse>> + Send + 'b
+            where 'a: 'b, Self: 'b;
+
+        fn complete_activity_task<'a, 'b>(
+            &self,
+            task_token: TaskToken,
+            result: Option<Payloads>,
+        ) -> impl Future<Output = Result<RespondActivityTaskCompletedResponse>> + Send + 'b
+            where 'a: 'b, Self: 'b;
+
+        fn cancel_activity_task<'a, 'b>(
+            &self,
+            task_token: TaskToken,
+            details: Option<Payloads>,
+        ) -> impl Future<Output = Result<RespondActivityTaskCanceledResponse>> + Send + 'b
+            where 'a: 'b, Self: 'b;
+
+        fn fail_activity_task<'a, 'b>(
+            &self,
+            task_token: TaskToken,
+            failure: Option<Failure>,
+        ) -> impl Future<Output = Result<RespondActivityTaskFailedResponse>> + Send + 'b
+            where 'a: 'b, Self: 'b;
+
+        fn fail_workflow_task<'a, 'b>(
+            &self,
+            task_token: TaskToken,
+            cause: WorkflowTaskFailedCause,
+            failure: Option<Failure>,
+        ) -> impl Future<Output = Result<RespondWorkflowTaskFailedResponse>> + Send + 'b
+            where 'a: 'b, Self: 'b;
+
+        fn signal_workflow_execution<'a, 'b>(
+            &self,
+            workflow_id: String,
+            run_id: String,
+            signal_name: String,
+            payloads: Option<Payloads>,
+        ) -> impl Future<Output = Result<SignalWorkflowExecutionResponse>> + Send + 'b
+            where 'a: 'b, Self: 'b;
+
+        fn record_activity_heartbeat<'a, 'b>(
+           &self,
+           task_token: TaskToken,
+           details: Option<Payloads>,
+        ) -> impl Future<Output = Result<RecordActivityTaskHeartbeatResponse>> + Send + 'b
+            where 'a: 'b, Self: 'b;
+
+        fn query_workflow_execution<'a, 'b>(
+            &self,
+            workflow_id: String,
+            run_id: String,
+            query: WorkflowQuery,
+        ) -> impl Future<Output = Result<QueryWorkflowResponse>> + Send + 'b
+            where 'a: 'b, Self: 'b;
+
+        fn describe_workflow_execution<'a, 'b>(
+            &self,
+            workflow_id: String,
+            run_id: Option<String>,
+        ) -> impl Future<Output = Result<DescribeWorkflowExecutionResponse>> + Send + 'b
+            where 'a: 'b, Self: 'b;
+
+        fn get_workflow_execution_history<'a, 'b>(
+            &self,
+            workflow_id: String,
+            run_id: Option<String>,
+            page_token: Vec<u8>
+        ) -> impl Future<Output = Result<GetWorkflowExecutionHistoryResponse>> + Send + 'b
+            where 'a: 'b, Self: 'b;
+
+        fn respond_legacy_query<'a, 'b>(
+            &self,
+            task_token: TaskToken,
+            query_result: QueryResult,
+        ) -> impl Future<Output = Result<RespondQueryTaskCompletedResponse>> + Send + 'b
+            where 'a: 'b, Self: 'b;
+
+        fn cancel_workflow_execution<'a, 'b>(
+            &self,
+            workflow_id: String,
+            run_id: Option<String>,
+        ) -> impl Future<Output = Result<RequestCancelWorkflowExecutionResponse>> + Send + 'b
+            where 'a: 'b, Self: 'b;
+
+        fn terminate_workflow_execution<'a, 'b>(
+            &self,
+            workflow_id: String,
+            run_id: Option<String>,
+        ) -> impl Future<Output = Result<TerminateWorkflowExecutionResponse>> + Send + 'b
+            where 'a: 'b, Self: 'b;
+
+        fn list_namespaces<'a, 'b>(
+            &self,
+        ) -> impl Future<Output = Result<ListNamespacesResponse>> + Send + 'b
+            where 'a: 'b, Self: 'b;
+
+        fn get_options(&self) -> &ServerGatewayOptions;
+    }
+}

package/sdk-core/{src/pollers → client/src}/retry.rs

@@ -1,11 +1,4 @@
-use crate::{
-    pollers::{
-        gateway::{RetryConfig, ServerGatewayApis},
-        Result, RETRYABLE_ERROR_CODES,
-    },
-    protosext::WorkflowTaskCompletion,
-    task_token::TaskToken,
-};
+use crate::{Result, RetryConfig, ServerGatewayApis, ServerGatewayOptions, WorkflowTaskCompletion};
 use backoff::{backoff::Backoff, ExponentialBackoff};
 use futures_retry::{ErrorHandler, FutureRetry, RetryPolicy};
 use std::{fmt::Debug, future::Future, time::Duration};
@@ -15,9 +8,21 @@ use temporal_sdk_core_protos::{
         common::v1::Payloads, enums::v1::WorkflowTaskFailedCause, failure::v1::Failure,
         query::v1::WorkflowQuery, workflowservice::v1::*,
     },
+    TaskToken,
 };
 use tonic::Code;
 
+/// List of gRPC error codes that client will retry.
+pub const RETRYABLE_ERROR_CODES: [Code; 7] = [
+    Code::DataLoss,
+    Code::Internal,
+    Code::Unknown,
+    Code::ResourceExhausted,
+    Code::Aborted,
+    Code::OutOfRange,
+    Code::Unavailable,
+];
+
 #[derive(Debug)]
 /// A wrapper for a [ServerGatewayApis] implementor which performs auto-retries
 pub struct RetryGateway<SG> {
@@ -103,18 +108,18 @@ impl ErrorHandler<tonic::Status> for TonicErrorHandler {
             return RetryPolicy::ForwardError(e);
         }
 
-        if current_attempt == 1 {
-            debug!(error=?e, "gRPC call {} failed on first attempt", self.call_name);
-        } else if self.should_log_retry_warning(current_attempt) {
-            warn!(error=?e, "gRPC call {} retried {} times", self.call_name, current_attempt);
-        }
-
         // Long polls are OK with being cancelled or running into the timeout because there's
         // nothing to do but retry anyway
         let long_poll_allowed = self.call_type == CallType::LongPoll
             && [Code::Cancelled, Code::DeadlineExceeded].contains(&e.code());
 
         if RETRYABLE_ERROR_CODES.contains(&e.code()) || long_poll_allowed {
+            if current_attempt == 1 {
+                debug!(error=?e, "gRPC call {} failed on first attempt", self.call_name);
+            } else if self.should_log_retry_warning(current_attempt) {
+                warn!(error=?e, "gRPC call {} retried {} times", self.call_name, current_attempt);
+            }
+
             match self.backoff.next_backoff() {
                 None => RetryPolicy::ForwardError(e), // None is returned when we've ran out of time
                 Some(backoff) => {
@@ -395,4 +400,157 @@ impl<SG: ServerGatewayApis + Send + Sync + 'static> ServerGatewayApis for RetryG
     async fn list_namespaces(&self) -> Result<ListNamespacesResponse> {
         retry_call!(self, CallType::Normal, list_namespaces,)
     }
+
+    fn get_options(&self) -> &ServerGatewayOptions {
+        self.gateway.get_options()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::MockServerGatewayApis;
+    use tonic::Status;
+
+    #[tokio::test]
+    async fn non_retryable_errors() {
+        for code in [
+            Code::InvalidArgument,
+            Code::NotFound,
+            Code::AlreadyExists,
+            Code::PermissionDenied,
+            Code::FailedPrecondition,
+            Code::Cancelled,
+            Code::DeadlineExceeded,
+            Code::Unauthenticated,
+            Code::Unimplemented,
+        ] {
+            let mut mock_gateway = MockServerGatewayApis::new();
+            mock_gateway
+                .expect_cancel_activity_task()
+                .returning(move |_, _| Err(Status::new(code, "non-retryable failure")))
+                .times(1);
+            let retry_gateway = RetryGateway::new(mock_gateway, Default::default());
+            let result = retry_gateway
+                .cancel_activity_task(vec![1].into(), None)
+                .await;
+            // Expecting an error after a single attempt, since there was a non-retryable error.
+            assert!(result.is_err());
+        }
+    }
+
+    #[tokio::test]
+    async fn long_poll_non_retryable_errors() {
+        for code in [
+            Code::InvalidArgument,
+            Code::NotFound,
+            Code::AlreadyExists,
+            Code::PermissionDenied,
+            Code::FailedPrecondition,
+            Code::Unauthenticated,
+            Code::Unimplemented,
+        ] {
+            let mut mock_gateway = MockServerGatewayApis::new();
+            mock_gateway
+                .expect_poll_workflow_task()
+                .returning(move |_, _| Err(Status::new(code, "non-retryable failure")))
+                .times(1);
+            mock_gateway
+                .expect_poll_activity_task()
+                .returning(move |_| Err(Status::new(code, "non-retryable failure")))
+                .times(1);
+            let retry_gateway = RetryGateway::new(mock_gateway, Default::default());
+            let result = retry_gateway
+                .poll_workflow_task("tq".to_string(), false)
+                .await;
+            assert!(result.is_err());
+            let result = retry_gateway.poll_activity_task("tq".to_string()).await;
+            assert!(result.is_err());
+        }
+    }
+
+    #[tokio::test]
+    async fn retryable_errors() {
+        for code in RETRYABLE_ERROR_CODES {
+            let mut mock_gateway = MockServerGatewayApis::new();
+            mock_gateway
+                .expect_cancel_activity_task()
+                .returning(move |_, _| Err(Status::new(code, "retryable failure")))
+                .times(3);
+            mock_gateway
+                .expect_cancel_activity_task()
+                .returning(|_, _| Ok(Default::default()))
+                .times(1);
+
+            let retry_gateway = RetryGateway::new(mock_gateway, Default::default());
+            let result = retry_gateway
+                .cancel_activity_task(vec![1].into(), None)
+                .await;
+            // Expecting successful response after retries
+            assert!(result.is_ok());
+        }
+    }
+
+    #[tokio::test]
+    async fn long_poll_retries_forever() {
+        let mut mock_gateway = MockServerGatewayApis::new();
+        mock_gateway
+            .expect_poll_workflow_task()
+            .returning(move |_, _| Err(Status::new(Code::Unknown, "retryable failure")))
+            .times(50);
+        mock_gateway
+            .expect_poll_workflow_task()
+            .returning(|_, _| Ok(Default::default()))
+            .times(1);
+        mock_gateway
+            .expect_poll_activity_task()
+            .returning(move |_| Err(Status::new(Code::Unknown, "retryable failure")))
+            .times(50);
+        mock_gateway
+            .expect_poll_activity_task()
+            .returning(|_| Ok(Default::default()))
+            .times(1);
+
+        let retry_gateway = RetryGateway::new(mock_gateway, Default::default());
+
+        let result = retry_gateway
+            .poll_workflow_task("tq".to_string(), false)
+            .await;
+        assert!(result.is_ok());
+        let result = retry_gateway.poll_activity_task("tq".to_string()).await;
+        assert!(result.is_ok());
+    }
+
+    #[tokio::test]
+    async fn long_poll_retries_deadline_exceeded() {
+        // For some reason we will get cancelled in these situations occasionally (always?) too
+        for code in [Code::Cancelled, Code::DeadlineExceeded] {
+            let mut mock_gateway = MockServerGatewayApis::new();
+            mock_gateway
+                .expect_poll_workflow_task()
+                .returning(move |_, _| Err(Status::new(code, "retryable failure")))
+                .times(5);
+            mock_gateway
+                .expect_poll_workflow_task()
+                .returning(|_, _| Ok(Default::default()))
+                .times(1);
+            mock_gateway
+                .expect_poll_activity_task()
+                .returning(move |_| Err(Status::new(code, "retryable failure")))
+                .times(5);
+            mock_gateway
+                .expect_poll_activity_task()
+                .returning(|_| Ok(Default::default()))
+                .times(1);
+
+            let retry_gateway = RetryGateway::new(mock_gateway, Default::default());
+
+            let result = retry_gateway
+                .poll_workflow_task("tq".to_string(), false)
+                .await;
+            assert!(result.is_ok());
+            let result = retry_gateway.poll_activity_task("tq".to_string()).await;
+            assert!(result.is_ok());
+        }
+    }
 }

package/sdk-core/core/Cargo.toml

@@ -0,0 +1,96 @@
+[package]
+name = "temporal-sdk-core"
+version = "0.1.0"
+authors = ["Spencer Judge <spencer@temporal.io>", "Vitaly Arbuzov <vitaly@temporal.io>"]
+edition = "2021"
+license-file = "LICENSE.txt"
+description = "Library for building new Temporal SDKs"
+homepage = "https://temporal.io/"
+repository = "https://github.com/temporalio/sdk-core"
+keywords = ["temporal", "workflow"]
+categories = ["development-tools"]
+
+[lib]
+
+[dependencies]
+anyhow = "1.0"
+arc-swap = "1.3"
+async-trait = "0.1"
+base64 = "0.13"
+crossbeam = "0.8"
+dashmap = "5.0"
+derive_builder = "0.10"
+derive_more = "0.99"
+enum_dispatch = "0.3"
+futures = "0.3"
+http = "0.2"
+hyper = "0.14"
+itertools = "0.10"
+lazy_static = "1.4"
+log = "0.4"
+lru = "0.7"
+once_cell = "1.5"
+opentelemetry = { version = "0.16", features = ["rt-tokio"] }
+opentelemetry-otlp = { version = "0.9.0", features = ["tokio", "metrics"] }
+opentelemetry-prometheus = "0.9.0"
+parking_lot = { version = "0.11", features = ["send_guard"] }
+prometheus = "0.12"
+prost = "0.9"
+prost-types = "0.9"
+rand = "0.8.3"
+ringbuf = "0.2"
+serde = "1.0"
+slotmap = "1.0"
+thiserror = "1.0"
+tokio = { version = "1.1", features = ["rt", "rt-multi-thread", "parking_lot", "time", "fs"] }
+tokio-util = { version = "0.6.9" }
+tokio-stream = "0.1"
+tonic = { version = "0.6", features = ["tls", "tls-roots"] }
+tracing = { version = "0.1", features = ["log-always"] }
+tracing-futures = "0.2"
+tracing-opentelemetry = "0.16"
+tracing-subscriber = { version = "0.3", features = ["parking_lot", "env-filter"] }
+url = "2.2"
+uuid = { version = "0.8.2", features = ["v4"] }
+
+# 1st party local deps
+[dependencies.temporal-sdk-core-api]
+path = "../core-api"
+version = "0.1"
+
+[dependencies.temporal-sdk-core-protos]
+path = "../sdk-core-protos"
+version = "0.1"
+features = ["history_builders"]
+
+[dependencies.temporal-client]
+path = "../client"
+version = "0.1"
+features = ["mocks"]
+
+[dependencies.rustfsm]
+path = "../fsm"
+version = "0.1"
+
+[dev-dependencies]
+assert_matches = "1.4"
+bimap = "0.6.1"
+mockall = "0.11"
+rstest = "0.12"
+temporal-sdk-core-test-utils = { path = "../test-utils" }
+temporal-sdk = { path = "../sdk" }
+
+[build-dependencies]
+tonic-build = "0.6"
+
+[[test]]
+name = "integ_tests"
+path = "../tests/main.rs"
+# Prevents autodiscovery, and hence these getting run with `cargo test`. Run with
+# `cargo test --test integ_tests`
+test = false
+
+[[test]]
+name = "load_tests"
+path = "../tests/load_tests.rs"
+test = false