@temporalio/core-bridge 1.10.3 → 1.11.1

package/sdk-core/client/src/retry.rs

@@ -146,16 +146,26 @@ where
             throttle_backoff: throttle_cfg.into_exp_backoff(throttle_clock),
         }
     }
-    const fn should_log_retry_warning(&self, cur_attempt: usize) -> bool {
+
+    fn maybe_log_retry(&self, cur_attempt: usize, err: &tonic::Status) {
+        let mut do_log = false;
         // Warn on more than 5 retries for unlimited retrying
         if self.max_retries == 0 && cur_attempt > 5 {
-            return true;
+            do_log = true;
         }
         // Warn if the attempts are more than 50% of max retries
         if self.max_retries > 0 && cur_attempt * 2 >= self.max_retries {
-            return true;
+            do_log = true;
+        }
+
+        if do_log {
+            // Error if unlimited retries have been going on for a while
+            if self.max_retries == 0 && cur_attempt > 15 {
+                error!(error=?err, "gRPC call {} retried {} times", self.call_name, cur_attempt);
+            } else {
+                warn!(error=?err, "gRPC call {} retried {} times", self.call_name, cur_attempt);
+            }
         }
-        false
     }
 }
 #[doc(hidden)]
@@ -194,8 +204,8 @@ where
         if RETRYABLE_ERROR_CODES.contains(&e.code()) || long_poll_allowed {
             if current_attempt == 1 {
                 debug!(error=?e, "gRPC call {} failed on first attempt", self.call_name);
-            } else if self.should_log_retry_warning(current_attempt) {
-                warn!(error=?e, "gRPC call {} retried {} times", self.call_name, current_attempt);
+            } else {
+                self.maybe_log_retry(current_attempt, &e);
             }
 
             match self.backoff.next_backoff() {
@@ -471,7 +481,7 @@ where
             list_open_workflow_executions,
             maximum_page_size,
             next_page_token.clone(),
-            start_time_filter.clone(),
+            start_time_filter,
             filters.clone()
         )
     }
@@ -488,7 +498,7 @@ where
             list_closed_workflow_executions,
             maximum_page_size,
             next_page_token.clone(),
-            start_time_filter.clone(),
+            start_time_filter,
             filters.clone()
         )
     }
@@ -541,7 +551,7 @@ where
             workflow_id.clone(),
             run_id.clone(),
             name.clone(),
-            wait_policy.clone(),
+            wait_policy,
             args.clone()
         )
     }

package/sdk-core/core/Cargo.toml

@@ -17,16 +17,16 @@ default = ["otel"]
 otel = ["dep:opentelemetry", "dep:opentelemetry_sdk", "dep:opentelemetry-otlp",
     "dep:opentelemetry-prometheus", "dep:hyper", "dep:hyper-util", "dep:http-body-util"]
 tokio-console = ["console-subscriber"]
-ephemeral-server = ["dep:flate2", "dep:nix", "dep:reqwest", "dep:tar", "dep:zip"]
+ephemeral-server = ["dep:flate2", "dep:reqwest", "dep:tar", "dep:zip"]
 
 [dependencies]
 anyhow = "1.0"
 async-trait = "0.1"
-console-subscriber = { version = "0.2", optional = true }
+console-subscriber = { version = "0.4", optional = true }
 crossbeam-channel = "0.5"
 crossbeam-queue = "0.3"
 crossbeam-utils = "0.8"
-dashmap = "5.5"
+dashmap = "6.0"
 derive_builder = { workspace = true }
 derive_more = { workspace = true }
 enum_dispatch = "0.3"
@@ -38,23 +38,22 @@ governor = "0.6"
 http-body-util = { version = "0.1", optional = true }
 hyper = { version = "1.2", optional = true }
 hyper-util = { version = "0.1", features = ["server", "http1", "http2", "tokio"], optional = true }
-itertools = "0.12"
+itertools = "0.13"
 lru = "0.12"
 mockall = "0.12"
-nix = { version = "0.28", optional = true, features = ["process", "signal"] }
 once_cell = { workspace = true }
 opentelemetry = { workspace = true, features = ["metrics"], optional = true }
-opentelemetry_sdk = { version = "0.22", features = ["rt-tokio", "metrics"], optional = true }
-opentelemetry-otlp = { version = "0.15", features = ["tokio", "metrics"], optional = true }
-opentelemetry-prometheus = { version = "0.15", optional = true }
+opentelemetry_sdk = { version = "0.24", features = ["rt-tokio", "metrics"], optional = true }
+opentelemetry-otlp = { version = "0.17", features = ["tokio", "metrics"], optional = true }
+opentelemetry-prometheus = { version = "0.17", optional = true }
 parking_lot = { version = "0.12", features = ["send_guard"] }
 pid = "4.0"
 pin-project = "1.0"
 prometheus = "0.13"
 prost = { workspace = true }
-prost-types = { version = "0.5", package = "prost-wkt-types" }
+prost-types = { version = "0.6", package = "prost-wkt-types" }
 rand = "0.8.3"
-reqwest = { version = "0.11", features = ["json", "stream", "rustls-tls"], default-features = false, optional = true }
+reqwest = { version = "0.12", features = ["json", "stream", "rustls-tls"], default-features = false, optional = true }
 ringbuf = "0.4"
 serde = "1.0"
 serde_json = "1.0"
@@ -71,7 +70,7 @@ tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["parking_lot", "env-filter", "registry"] }
 url = "2.2"
 uuid = { version = "1.1", features = ["v4"] }
-zip = { version = "1.2", optional = true }
+zip = { version = "2.0", optional = true }
 
 # 1st party local deps
 [dependencies.temporal-sdk-core-api]
@@ -93,10 +92,11 @@ assert_matches = "1.4"
 bimap = "0.6.1"
 clap = { version = "4.0", features = ["derive"] }
 criterion = "0.5"
-rstest = "0.19.0"
+rstest = "0.21"
 sysinfo = "0.30"
 temporal-sdk-core-test-utils = { path = "../test-utils" }
 temporal-sdk = { path = "../sdk" }
+tokio-stream = { version = "0.1", features = ["net"] }
 
 [build-dependencies]
 tonic-build = { workspace = true }

package/sdk-core/core/src/abstractions.rs

@@ -66,7 +66,7 @@ where
             .map(|ap| ap + self.unused_claimants.load(Ordering::Acquire))
     }
 
-    pub(crate) async fn acquire_owned(&self) -> Result<OwnedMeteredSemPermit<SK>, ()> {
+    pub(crate) async fn acquire_owned(&self) -> OwnedMeteredSemPermit<SK> {
         if let Some(max) = self.max_permits {
             self.extant_permits
                 .1
@@ -76,7 +76,7 @@ where
                 .expect("Extant permit channel is never closed");
         }
         let res = self.supplier.reserve_slot(self).await;
-        Ok(self.build_owned(res))
+        self.build_owned(res)
     }
 
     pub(crate) fn try_acquire_owned(&self) -> Result<OwnedMeteredSemPermit<SK>, ()> {
@@ -351,6 +351,6 @@ pub(crate) mod tests {
         advance_fut!(acquire_fut);
         drop(perm);
         // Now it'll proceed
-        acquire_fut.await.unwrap();
+        acquire_fut.await;
     }
 }

package/sdk-core/core/src/core_tests/activity_tasks.rs

@@ -900,7 +900,7 @@ async fn activity_tasks_from_completion_reserve_slots() {
         ],
         mock,
     );
-    mh.completion_asserts = Some(Box::new(|wftc| {
+    mh.completion_mock_fn = Some(Box::new(|wftc| {
         // Make sure when we see the completion with the schedule act command that it does
         // not have the eager execution flag set the first time, and does the second.
         if let Some(Attributes::ScheduleActivityTaskCommandAttributes(attrs)) = wftc
@@ -914,6 +914,7 @@ async fn activity_tasks_from_completion_reserve_slots() {
                 assert!(attrs.request_eager_execution);
             }
         }
+        Ok(Default::default())
     }));
     let mut mock = build_mock_pollers(mh);
     mock.worker_cfg(|cfg| {

package/sdk-core/core/src/core_tests/determinism.rs

@@ -125,7 +125,7 @@ async fn activity_id_or_type_change_is_nondeterministic(
 ) {
     let wf_id = "fakeid";
     let wf_type = DEFAULT_WORKFLOW_TYPE;
-    let mut t = if local_act {
+    let mut t: TestHistoryBuilder = if local_act {
         canned_histories::single_local_activity("1")
     } else {
         canned_histories::single_activity("1")

package/sdk-core/core/src/core_tests/local_activities.rs

@@ -17,12 +17,11 @@ use std::{
         atomic::{AtomicUsize, Ordering},
         Arc,
     },
-    time::{Duration, SystemTime},
+    time::{Duration, Instant, SystemTime},
 };
 use temporal_client::WorkflowOptions;
 use temporal_sdk::{
-    ActContext, ActivityCancelledError, LocalActivityOptions, WfContext, WorkflowFunction,
-    WorkflowResult,
+    ActContext, ActivityError, LocalActivityOptions, WfContext, WorkflowFunction, WorkflowResult,
 };
 use temporal_sdk_core_api::{
     errors::{PollActivityError, PollWfError},
@@ -49,7 +48,7 @@ use temporal_sdk_core_test_utils::{
 };
 use tokio::{join, select, sync::Barrier};
 
-async fn echo(_ctx: ActContext, e: String) -> anyhow::Result<String> {
+async fn echo(_ctx: ActContext, e: String) -> Result<String, ActivityError> {
     Ok(e)
 }
 
@@ -279,7 +278,7 @@ async fn local_act_fail_and_retry(#[case] eventually_pass: bool) {
         if 2 == attempts.fetch_add(1, Ordering::Relaxed) && eventually_pass {
             Ok(())
         } else {
-            Err(anyhow!("Oh no I failed!"))
+            Err(anyhow!("Oh no I failed!").into())
         }
     });
     worker
@@ -356,7 +355,7 @@ async fn local_act_retry_long_backoff_uses_timer() {
     worker.register_activity(
         DEFAULT_ACTIVITY_TYPE,
         move |_ctx: ActContext, _: String| async move {
-            Result::<(), _>::Err(anyhow!("Oh no I failed!"))
+            Result::<(), _>::Err(anyhow!("Oh no I failed!").into())
         },
     );
     worker
@@ -928,7 +927,7 @@ async fn start_to_close_timeout_allows_retries(#[values(true, false)] la_complet
                     _ = tokio::time::sleep(Duration::from_millis(100)) => (),
                     _ = ctx.cancelled() => {
                         cancels.fetch_add(1, Ordering::AcqRel);
-                        return Err(anyhow!(ActivityCancelledError::default()));
+                        return Err(ActivityError::cancelled());
                     }
                 }
             }
@@ -991,7 +990,7 @@ async fn wft_failure_cancels_running_las() {
             if res.is_err() {
                 panic!("Activity must be cancelled!!!!");
             }
-            Result::<(), _>::Err(ActivityCancelledError::default().into())
+            Result::<(), _>::Err(ActivityError::cancelled())
         },
     );
     worker
@@ -1076,11 +1075,12 @@ async fn local_act_records_nonfirst_attempts_ok() {
     let mut mh = MockPollCfg::from_resp_batches(wf_id, t, [1, 2, 3], mock);
     let nonfirst_counts = Arc::new(SegQueue::new());
     let nfc_c = nonfirst_counts.clone();
-    mh.completion_asserts = Some(Box::new(move |c| {
+    mh.completion_mock_fn = Some(Box::new(move |c| {
         nfc_c.push(
             c.metering_metadata
                 .nonfirst_local_activity_execution_attempts,
         );
+        Ok(Default::default())
     }));
     let mut worker = mock_sdk_cfg(mh, |wc| {
         wc.max_cached_workflows = 1;
@@ -1107,7 +1107,7 @@ async fn local_act_records_nonfirst_attempts_ok() {
         },
     );
     worker.register_activity("echo", move |_ctx: ActContext, _: String| async move {
-        Result::<(), _>::Err(anyhow!("I fail"))
+        Result::<(), _>::Err(anyhow!("I fail").into())
     });
     worker
         .submit_wf(
@@ -1390,3 +1390,66 @@ async fn local_activity_after_wf_complete_is_discarded() {
     join!(wf_poller, at_poller);
     core.drain_pollers_and_shutdown().await;
 }
+
+#[tokio::test]
+async fn local_act_retry_explicit_delay() {
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_workflow_task_scheduled_and_started();
+
+    let wf_id = "fakeid";
+    let mock = mock_workflow_client();
+    let mh = MockPollCfg::from_resp_batches(wf_id, t, [1], mock);
+    let mut worker = mock_sdk(mh);
+
+    worker.register_wf(
+        DEFAULT_WORKFLOW_TYPE.to_owned(),
+        move |ctx: WfContext| async move {
+            let la_res = ctx
+                .local_activity(LocalActivityOptions {
+                    activity_type: "echo".to_string(),
+                    input: "hi".as_json_payload().expect("serializes fine"),
+                    retry_policy: RetryPolicy {
+                        initial_interval: Some(prost_dur!(from_millis(50))),
+                        backoff_coefficient: 1.0,
+                        maximum_attempts: 5,
+                        ..Default::default()
+                    },
+                    ..Default::default()
+                })
+                .await;
+            assert!(la_res.completed_ok());
+            Ok(().into())
+        },
+    );
+    let attempts: &'static _ = Box::leak(Box::new(AtomicUsize::new(0)));
+    worker.register_activity("echo", move |_ctx: ActContext, _: String| async move {
+        // Succeed on 3rd attempt (which is ==2 since fetch_add returns prev val)
+        let last_attempt = attempts.fetch_add(1, Ordering::Relaxed);
+        if 0 == last_attempt {
+            Err(ActivityError::Retryable {
+                source: anyhow!("Explicit backoff error"),
+                explicit_delay: Some(Duration::from_millis(300)),
+            })
+        } else if 2 == last_attempt {
+            Ok(())
+        } else {
+            Err(anyhow!("Oh no I failed!").into())
+        }
+    });
+    worker
+        .submit_wf(
+            wf_id.to_owned(),
+            DEFAULT_WORKFLOW_TYPE.to_owned(),
+            vec![],
+            WorkflowOptions::default(),
+        )
+        .await
+        .unwrap();
+    let start = Instant::now();
+    worker.run_until_done().await.unwrap();
+    let expected_attempts = 3;
+    assert_eq!(expected_attempts, attempts.load(Ordering::Relaxed));
+    // There will be one 300ms backoff and one 50s backoff, so things should take at least that long
+    assert!(start.elapsed() > Duration::from_millis(350));
+}

package/sdk-core/core/src/core_tests/queries.rs

@@ -167,7 +167,7 @@ async fn new_queries(#[values(1, 3)] num_queries: usize) {
     let mut mock_client = mock_workflow_client();
     mock_client.expect_respond_legacy_query().times(0);
     let mut mh = MockPollCfg::from_resp_batches(wfid, t, tasks, mock_workflow_client());
-    mh.completion_asserts = Some(Box::new(move |c| {
+    mh.completion_mock_fn = Some(Box::new(move |c| {
         // If the completion is the one ending the workflow, make sure it includes the query resps
         if c.commands[0].command_type() == CommandType::CompleteWorkflowExecution {
             assert_eq!(c.query_responses.len(), num_queries);
@@ -176,6 +176,7 @@ async fn new_queries(#[values(1, 3)] num_queries: usize) {
         } else {
             panic!("Unexpected command in response")
         }
+        Ok(Default::default())
     }));
     let mut mock = build_mock_pollers(mh);
     mock.worker_cfg(|wc| wc.max_cached_workflows = 10);

package/sdk-core/core/src/core_tests/updates.rs

@@ -1,14 +1,28 @@
-use crate::test_help::{build_mock_pollers, mock_worker, MockPollCfg, ResponseType};
+use crate::{
+    test_help::{
+        build_mock_pollers, hist_to_poll_resp, mock_worker, MockPollCfg, PollWFTRespExt,
+        ResponseType,
+    },
+    worker::client::mocks::mock_workflow_client,
+};
 use temporal_sdk_core_api::Worker;
 use temporal_sdk_core_protos::{
     coresdk::{
         workflow_activation::{workflow_activation_job, WorkflowActivationJob},
-        workflow_commands::{update_response::Response, CompleteWorkflowExecution, UpdateResponse},
+        workflow_commands::{
+            update_response::Response, CompleteWorkflowExecution, ScheduleActivity, UpdateResponse,
+        },
         workflow_completion::WorkflowActivationCompletion,
     },
-    temporal::api::{common::v1::Payload, enums::v1::EventType},
-    TestHistoryBuilder,
+    temporal::api::{
+        common::v1::Payload,
+        enums::v1::EventType,
+        update::v1::{Acceptance, Rejection},
+        workflowservice::v1::RespondWorkflowTaskCompletedResponse,
+    },
+    TestHistoryBuilder, DEFAULT_ACTIVITY_TYPE,
 };
+use temporal_sdk_core_test_utils::WorkerTestHelpers;
 
 #[tokio::test]
 async fn replay_with_empty_first_task() {
@@ -71,3 +85,147 @@ async fn replay_with_empty_first_task() {
     .await
     .unwrap();
 }
+
+#[rstest::rstest]
+#[tokio::test]
+async fn initial_request_sent_back(#[values(false, true)] reject: bool) {
+    let wfid = "fakeid";
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_workflow_task_scheduled_and_started();
+
+    let update_id = "upd-1";
+    let mut poll_resp = hist_to_poll_resp(&t, wfid, ResponseType::AllHistory);
+    let upd_req_body = poll_resp.add_update_request(update_id, 1);
+
+    let mut mock_client = mock_workflow_client();
+    mock_client
+        .expect_complete_workflow_task()
+        .times(1)
+        .returning(move |mut resp| {
+            let msg = resp.messages.pop().unwrap();
+            let orig_req = if reject {
+                let acceptance = msg.body.unwrap().unpack_as(Rejection::default()).unwrap();
+                acceptance.rejected_request.unwrap()
+            } else {
+                let acceptance = msg.body.unwrap().unpack_as(Acceptance::default()).unwrap();
+                acceptance.accepted_request.unwrap()
+            };
+            assert_eq!(orig_req, upd_req_body);
+            Ok(RespondWorkflowTaskCompletedResponse::default())
+        });
+    let mh = MockPollCfg::from_resp_batches(wfid, t, [poll_resp], mock_client);
+    let mut mock = build_mock_pollers(mh);
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 1);
+    let core = mock_worker(mock);
+
+    let task = core.poll_workflow_activation().await.unwrap();
+    let resp = if reject {
+        Response::Rejected(Default::default())
+    } else {
+        Response::Accepted(())
+    };
+    core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
+        task.run_id,
+        UpdateResponse {
+            protocol_instance_id: update_id.to_string(),
+            response: Some(resp),
+        }
+        .into(),
+    ))
+    .await
+    .unwrap();
+}
+
+#[tokio::test]
+async fn speculative_wft_with_command_event() {
+    let wfid = "fakeid";
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task();
+    t.add_activity_task_scheduled("act1");
+
+    let mut spec_task_hist = t.clone();
+    spec_task_hist.add_workflow_task_scheduled_and_started();
+
+    let mut real_hist = t.clone();
+    real_hist.add_we_signaled("hi", vec![]);
+    real_hist.add_workflow_task_scheduled_and_started();
+
+    let update_id = "upd-1";
+    let mut speculative_task = hist_to_poll_resp(&spec_task_hist, wfid, ResponseType::OneTask(2));
+    speculative_task.add_update_request(update_id, 1);
+    // Verify the speculative task contains the activity scheduled event
+    assert_eq!(
+        speculative_task.history.as_ref().unwrap().events[1].event_type,
+        EventType::ActivityTaskScheduled as i32
+    );
+
+    let mock_client = mock_workflow_client();
+    let mut mh = MockPollCfg::from_resp_batches(
+        wfid,
+        real_hist,
+        [
+            ResponseType::ToTaskNum(1),
+            speculative_task.into(),
+            ResponseType::OneTask(3),
+        ],
+        mock_client,
+    );
+    let mut completes = 0;
+    mh.completion_mock_fn = Some(Box::new(move |_| {
+        completes += 1;
+        let mut r = RespondWorkflowTaskCompletedResponse::default();
+        if completes == 2 {
+            // The second response (the update rejection) needs to indicate that the last started
+            // wft ID should be reset.
+            r.reset_history_event_id = 3;
+        }
+        Ok(r)
+    }));
+    let mut mock = build_mock_pollers(mh);
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 1);
+    let core = mock_worker(mock);
+
+    let task = core.poll_workflow_activation().await.unwrap();
+    core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
+        task.run_id,
+        ScheduleActivity {
+            activity_id: "act1".to_string(),
+            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+            ..Default::default()
+        }
+        .into(),
+    ))
+    .await
+    .unwrap();
+
+    // Receive the task containing and reject the update
+    let task = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        task.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::DoUpdate(_)),
+        }]
+    );
+    core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
+        task.run_id,
+        UpdateResponse {
+            protocol_instance_id: update_id.to_string(),
+            response: Some(Response::Rejected(Default::default())),
+        }
+        .into(),
+    ))
+    .await
+    .unwrap();
+
+    // Now we'll get another task with the "real" history containing the signal
+    let task = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        task.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
+        }]
+    );
+    core.complete_execution(&task.run_id).await;
+}

package/sdk-core/core/src/core_tests/workers.rs

@@ -1,9 +1,10 @@
 use crate::{
     prost_dur,
     test_help::{
-        build_fake_worker, build_mock_pollers, canned_histories, mock_worker, MockPollCfg,
-        MockWorkerInputs, MocksHolder, ResponseType, WorkerExt,
+        build_fake_worker, build_mock_pollers, canned_histories, mock_worker, test_worker_cfg,
+        MockPollCfg, MockWorkerInputs, MocksHolder, ResponseType, WorkerExt,
     },
+    worker,
     worker::client::mocks::mock_workflow_client,
     PollActivityError, PollWfError,
 };
@@ -259,3 +260,38 @@ async fn worker_does_not_panic_on_retry_exhaustion_of_nonfatal_net_err() {
         Some(workflow_activation_job::Variant::RemoveFromCache(_))
     );
 }
+
+#[rstest::rstest]
+#[tokio::test]
+async fn worker_can_shutdown_after_never_polling_ok(#[values(true, false)] poll_workflow: bool) {
+    let mut mock = mock_workflow_client();
+    mock.expect_poll_activity_task()
+        .returning(|_, _| Err(tonic::Status::permission_denied("you shall not pass")));
+    if poll_workflow {
+        mock.expect_poll_workflow_task()
+            .returning(|_| Err(tonic::Status::permission_denied("you shall not pass")));
+    }
+    let core = worker::Worker::new_test(
+        test_worker_cfg()
+            .max_concurrent_at_polls(1_usize)
+            .build()
+            .unwrap(),
+        mock,
+    );
+
+    loop {
+        // Must continue polling until polls return shutdown.
+        if poll_workflow {
+            let res = core.poll_workflow_activation().await.unwrap_err();
+            if !matches!(res, PollWfError::ShutDown) {
+                continue;
+            }
+        }
+        let res = core.poll_activity_task().await.unwrap_err();
+        if !matches!(res, PollActivityError::ShutDown) {
+            continue;
+        }
+        core.finalize_shutdown().await;
+        break;
+    }
+}