@temporalio/core-bridge 1.5.2 → 1.7.0

package/sdk-core/core/src/core_tests/activity_tasks.rs

@@ -4,7 +4,7 @@ use crate::{
         build_fake_worker, build_mock_pollers, canned_histories, gen_assert_and_reply,
         mock_manual_poller, mock_poller, mock_poller_from_resps, mock_worker, poll_and_reply,
         single_hist_mock_sg, test_worker_cfg, MockPollCfg, MockWorkerInputs, MocksHolder,
-        ResponseType, WorkflowCachingPolicy, TEST_Q,
+        QueueResponse, ResponseType, WorkerExt, WorkflowCachingPolicy, TEST_Q,
     },
     worker::client::mocks::{mock_manual_workflow_client, mock_workflow_client},
     ActivityHeartbeat, Worker, WorkerConfigBuilder,
@@ -13,7 +13,8 @@ use futures::FutureExt;
 use itertools::Itertools;
 use std::{
     cell::RefCell,
-    collections::{hash_map::Entry, HashMap, VecDeque},
+    collections::{hash_map::Entry, HashMap, HashSet, VecDeque},
+    future,
     rc::Rc,
     sync::{
         atomic::{AtomicUsize, Ordering},
@@ -23,15 +24,17 @@ use std::{
 };
 use temporal_client::WorkflowOptions;
 use temporal_sdk::{ActivityOptions, WfContext};
-use temporal_sdk_core_api::{errors::CompleteActivityError, Worker as WorkerTrait};
-use temporal_sdk_core_protos::temporal::api::command::v1::ScheduleActivityTaskCommandAttributes;
+use temporal_sdk_core_api::{
+    errors::{CompleteActivityError, PollActivityError},
+    Worker as WorkerTrait,
+};
 use temporal_sdk_core_protos::{
     coresdk::{
         activity_result::{
             activity_execution_result, activity_resolution, ActivityExecutionResult,
             ActivityResolution, Success,
         },
-        activity_task::{activity_task, ActivityTask},
+        activity_task::{activity_task, ActivityCancelReason, ActivityTask, Cancel},
         workflow_activation::{workflow_activation_job, ResolveActivity, WorkflowActivationJob},
         workflow_commands::{
             ActivityCancellationType, CompleteWorkflowExecution, RequestCancelActivity,
@@ -41,8 +44,11 @@ use temporal_sdk_core_protos::{
         ActivityTaskCompletion,
     },
     temporal::api::{
-        command::v1::command::Attributes,
+        command::v1::{command::Attributes, ScheduleActivityTaskCommandAttributes},
         enums::v1::EventType,
+        history::v1::{
+            history_event::Attributes as EventAttributes, ActivityTaskScheduledEventAttributes,
+        },
         workflowservice::v1::{
             PollActivityTaskQueueResponse, RecordActivityTaskHeartbeatResponse,
             RespondActivityTaskCanceledResponse, RespondActivityTaskCompletedResponse,
@@ -53,11 +59,10 @@ use temporal_sdk_core_protos::{
 };
 use temporal_sdk_core_test_utils::{fanout_tasks, start_timer_cmd, TestWorker};
 use tokio::{sync::Barrier, time::sleep};
+use tokio_util::sync::CancellationToken;
 
-#[tokio::test]
-async fn max_activities_respected() {
-    let _task_q = "q";
-    let mut tasks = VecDeque::from(vec![
+fn three_tasks() -> VecDeque<PollActivityTaskQueueResponse> {
+    VecDeque::from(vec![
         PollActivityTaskQueueResponse {
             task_token: vec![1],
             activity_id: "act1".to_string(),
@@ -73,7 +78,13 @@ async fn max_activities_respected() {
             activity_id: "act3".to_string(),
             ..Default::default()
         },
-    ]);
+    ])
+}
+
+#[tokio::test]
+async fn max_activities_respected() {
+    let _task_q = "q";
+    let mut tasks = three_tasks();
     let mut mock_client = mock_workflow_client();
     mock_client
         .expect_poll_activity_task()
@@ -122,7 +133,7 @@ async fn activity_not_found_returns_ok() {
     })
     .await
     .unwrap();
-    core.shutdown().await;
+    core.drain_activity_poller_and_shutdown().await;
 }
 
 #[tokio::test]
@@ -218,12 +229,14 @@ async fn heartbeats_report_cancels_only_once() {
     })
     .await
     .unwrap();
-    core.shutdown().await;
+    core.drain_activity_poller_and_shutdown().await;
 }
 
 #[tokio::test]
 async fn activity_cancel_interrupts_poll() {
     let mut mock_poller = mock_manual_poller();
+    let shutdown_token = CancellationToken::new();
+    let shutdown_token_clone = shutdown_token.clone();
     let mut poll_resps = VecDeque::from(vec![
         async {
             Some(Ok(PollActivityTaskQueueResponse {
@@ -238,10 +251,15 @@ async fn activity_cancel_interrupts_poll() {
             Some(Ok(Default::default()))
         }
         .boxed(),
+        async move {
+            shutdown_token.cancelled().await;
+            None
+        }
+        .boxed(),
     ]);
     mock_poller
         .expect_poll()
-        .times(2)
+        .times(3)
         .returning(move || poll_resps.pop_front().unwrap());
 
     let mut mock_client = mock_manual_workflow_client();
@@ -290,11 +308,12 @@ async fn activity_cancel_interrupts_poll() {
                 }
             ).await.unwrap();
             last_finisher.store(2, Ordering::SeqCst);
+            shutdown_token_clone.cancel();
         }
     };
     // So that we know we blocked
     assert_eq!(last_finisher.load(Ordering::Acquire), 2);
-    core.shutdown().await;
+    core.drain_activity_poller_and_shutdown().await;
 }
 
 #[tokio::test]
@@ -343,13 +362,10 @@ async fn many_concurrent_heartbeat_cancels() {
             })
             .collect::<Vec<_>>(),
     );
-    // Because the mock is so fast, it's possible it can return before the cancel channel in
-    // the activity task poll selector. So, the final poll when there are no more tasks must
-    // take a while.
     poll_resps.push_back(
         async {
-            sleep(Duration::from_secs(10)).await;
-            unreachable!("Long poll")
+            future::pending::<()>().await;
+            unreachable!()
         }
         .boxed(),
     );
@@ -432,7 +448,7 @@ async fn many_concurrent_heartbeat_cancels() {
     })
     .await;
 
-    worker.shutdown().await;
+    worker.drain_activity_poller_and_shutdown().await;
 }
 
 #[tokio::test]
@@ -484,7 +500,7 @@ async fn activity_timeout_no_double_resolve() {
     )
     .await;
 
-    core.shutdown().await;
+    core.drain_pollers_and_shutdown().await;
 }
 
 #[tokio::test]
@@ -530,7 +546,7 @@ async fn can_heartbeat_acts_during_shutdown() {
     })
     .await
     .unwrap();
-    shutdown_fut.await;
+    core.drain_activity_poller_and_shutdown().await;
 }
 
 /// Verifies that if a user has tried to record a heartbeat and then immediately after failed the
@@ -581,7 +597,7 @@ async fn complete_act_with_fail_flushes_heartbeat() {
     })
     .await
     .unwrap();
-    core.shutdown().await;
+    core.drain_activity_poller_and_shutdown().await;
 
     // Verify the last seen call to record a heartbeat had the last detail payload
     let last_seen_payload = &last_seen_payload.take().unwrap().payloads[0];
@@ -654,6 +670,7 @@ async fn no_eager_activities_requested_when_worker_options_disable_remote_activi
             Ok(RespondWorkflowTaskCompletedResponse {
                 workflow_task: None,
                 activity_tasks: vec![],
+                reset_history_event_id: 0,
             })
         });
     let mut mock = single_hist_mock_sg(wfid, t, [1], mock, true);
@@ -686,7 +703,7 @@ async fn no_eager_activities_requested_when_worker_options_disable_remote_activi
     .await
     .unwrap();
 
-    core.shutdown().await;
+    core.drain_pollers_and_shutdown().await;
 
     assert_eq!(num_eager_requested.load(Ordering::Relaxed), 0);
 }
@@ -703,7 +720,7 @@ async fn activity_tasks_from_completion_are_delivered() {
     t.add_by_type(EventType::WorkflowExecutionStarted);
     t.add_full_wf_task();
     let act_same_queue_scheduled_ids = (1..4)
-        .map(|i| t.add_activity_task_scheduled(format!("act_id_{}_same_queue", i)))
+        .map(|i| t.add_activity_task_scheduled(format!("act_id_{i}_same_queue")))
         .collect_vec();
     t.add_activity_task_scheduled("act_id_same_queue_not_eager");
     t.add_activity_task_scheduled("act_id_different_queue");
@@ -742,21 +759,19 @@ async fn activity_tasks_from_completion_are_delivered() {
                 activity_tasks: (1..4)
                     .map(|i| PollActivityTaskQueueResponse {
                         task_token: vec![i],
-                        activity_id: format!("act_id_{}_same_queue", i),
+                        activity_id: format!("act_id_{i}_same_queue"),
                         ..Default::default()
                     })
                     .collect_vec(),
+                reset_history_event_id: 0,
             })
         });
     mock.expect_complete_activity_task()
         .times(3)
         .returning(|_, _| Ok(RespondActivityTaskCompletedResponse::default()));
     let mut mock = single_hist_mock_sg(wfid, t, [1], mock, true);
-    let mut mock_poller = mock_manual_poller();
-    mock_poller
-        .expect_poll()
-        .returning(|| futures::future::pending().boxed());
-    mock.set_act_poller(Box::new(mock_poller));
+    let act_tasks: Vec<QueueResponse<PollActivityTaskQueueResponse>> = vec![];
+    mock.set_act_poller(mock_poller_from_resps(act_tasks));
     mock.worker_cfg(|wc| wc.max_cached_workflows = 2);
     let core = mock_worker(mock);
 
@@ -766,7 +781,7 @@ async fn activity_tasks_from_completion_are_delivered() {
         .map(|seq| {
             ScheduleActivity {
                 seq,
-                activity_id: format!("act_id_{}_same_queue", seq),
+                activity_id: format!("act_id_{seq}_same_queue"),
                 task_queue: TEST_Q.to_string(),
                 cancellation_type: ActivityCancellationType::TryCancel as i32,
                 ..Default::default()
@@ -815,7 +830,7 @@ async fn activity_tasks_from_completion_are_delivered() {
         .unwrap();
     }
 
-    core.shutdown().await;
+    core.drain_pollers_and_shutdown().await;
 
     // Verify only a single eager activity was scheduled (the one on our worker's task queue)
     assert_eq!(num_eager_requested.load(Ordering::Relaxed), 3);
@@ -827,11 +842,23 @@ async fn activity_tasks_from_completion_reserve_slots() {
     let mut t = TestHistoryBuilder::default();
     t.add_by_type(EventType::WorkflowExecutionStarted);
     t.add_full_wf_task();
-    let schedid = t.add_activity_task_scheduled("1");
+    let schedid = t.add(EventAttributes::ActivityTaskScheduledEventAttributes(
+        ActivityTaskScheduledEventAttributes {
+            activity_id: "1".to_string(),
+            activity_type: Some("act1".into()),
+            ..Default::default()
+        },
+    ));
     let startid = t.add_activity_task_started(schedid);
     t.add_activity_task_completed(schedid, startid, b"hi".into());
     t.add_full_wf_task();
-    let schedid = t.add_activity_task_scheduled("2");
+    let schedid = t.add(EventAttributes::ActivityTaskScheduledEventAttributes(
+        ActivityTaskScheduledEventAttributes {
+            activity_id: "2".to_string(),
+            activity_type: Some("act2".into()),
+            ..Default::default()
+        },
+    ));
     let startid = t.add_activity_task_started(schedid);
     t.add_activity_task_completed(schedid, startid, b"hi".into());
     t.add_full_wf_task();
@@ -901,19 +928,25 @@ async fn activity_tasks_from_completion_reserve_slots() {
     // First poll for activities twice, occupying both slots
     let at1 = core.poll_activity_task().await.unwrap();
     let at2 = core.poll_activity_task().await.unwrap();
-
-    worker.register_wf(DEFAULT_WORKFLOW_TYPE, move |ctx: WfContext| async move {
-        ctx.activity(ActivityOptions {
-            activity_type: "act1".to_string(),
-            ..Default::default()
-        })
-        .await;
-        ctx.activity(ActivityOptions {
-            activity_type: "act2".to_string(),
-            ..Default::default()
-        })
-        .await;
-        Ok(().into())
+    let workflow_complete_token = CancellationToken::new();
+    let workflow_complete_token_clone = workflow_complete_token.clone();
+
+    worker.register_wf(DEFAULT_WORKFLOW_TYPE, move |ctx: WfContext| {
+        let complete_token = workflow_complete_token.clone();
+        async move {
+            ctx.activity(ActivityOptions {
+                activity_type: "act1".to_string(),
+                ..Default::default()
+            })
+            .await;
+            ctx.activity(ActivityOptions {
+                activity_type: "act2".to_string(),
+                ..Default::default()
+            })
+            .await;
+            complete_token.cancel();
+            Ok(().into())
+        }
     });
 
     worker
@@ -940,6 +973,13 @@ async fn activity_tasks_from_completion_reserve_slots() {
         .await
         .unwrap();
         barr.wait().await;
+        // Wait for workflow to complete in order for all eager activities to be requested before shutting down.
+        // After shutdown, no eager activities slots can be allocated.
+        workflow_complete_token_clone.cancelled().await;
+        core.initiate_shutdown();
+        // Even though this test requests eager activity tasks, none are returned in poll responses.
+        let err = core.poll_activity_task().await.unwrap_err();
+        assert_matches!(err, PollActivityError::ShutDown);
     };
     // This wf poll should *not* set the flag that it wants tasks back since both slots are
     // occupied
@@ -973,7 +1013,7 @@ async fn retryable_net_error_exhaustion_is_nonfatal() {
     })
     .await
     .unwrap();
-    core.shutdown().await;
+    core.drain_activity_poller_and_shutdown().await;
 }
 
 #[tokio::test]
@@ -1011,3 +1051,56 @@ async fn cant_complete_activity_with_unset_result_payload() {
         Err(CompleteActivityError::MalformedActivityCompletion { .. })
     )
 }
+
+#[tokio::test]
+async fn graceful_shutdown() {
+    let _task_q = "q";
+    let mut tasks = three_tasks();
+    let mut mock_client = mock_workflow_client();
+    mock_client
+        .expect_poll_activity_task()
+        .times(3)
+        .returning(move |_, _| Ok(tasks.pop_front().unwrap()));
+    // They shall all be reported as failed
+    mock_client
+        .expect_fail_activity_task()
+        .times(3)
+        .returning(|_, _| Ok(Default::default()));
+
+    let worker = Worker::new_test(
+        test_worker_cfg()
+            .graceful_shutdown_period(Duration::from_millis(500))
+            .build()
+            .unwrap(),
+        mock_client,
+    );
+
+    let _1 = worker.poll_activity_task().await.unwrap();
+    let _2 = worker.poll_activity_task().await.unwrap();
+    let _3 = worker.poll_activity_task().await.unwrap();
+
+    worker.initiate_shutdown();
+    let expected_tts = HashSet::from([vec![1], vec![2], vec![3]]);
+    let mut seen_tts = HashSet::new();
+    for _ in 1..=3 {
+        let cancel = worker.poll_activity_task().await.unwrap();
+        assert_matches!(
+            cancel.variant,
+            Some(activity_task::Variant::Cancel(Cancel {
+                reason: r
+            })) if r == ActivityCancelReason::WorkerShutdown as i32
+        );
+        seen_tts.insert(cancel.task_token);
+    }
+    assert_eq!(expected_tts, seen_tts);
+    for tt in seen_tts {
+        worker
+            .complete_activity_task(ActivityTaskCompletion {
+                task_token: tt,
+                result: Some(ActivityExecutionResult::cancel_from_details(None)),
+            })
+            .await
+            .unwrap();
+    }
+    worker.drain_pollers_and_shutdown().await;
+}

package/sdk-core/core/src/core_tests/child_workflows.rs

@@ -88,11 +88,12 @@ async fn parent_cancels_child_wf(ctx: WfContext) -> WorkflowResult<()> {
         .await
         .into_started()
         .expect("Child should get started");
-    let cancel_fut = start_res.cancel(&ctx);
-    let resfut = start_res.result();
-    let (cancel_res, res) = join!(cancel_fut, resfut);
-    cancel_res.expect("cancel result is ok");
-    let stat = res.status.expect("child wf result is ok");
+    start_res.cancel(&ctx);
+    let stat = start_res
+        .result()
+        .await
+        .status
+        .expect("child wf result is ok");
     assert_matches!(stat, child_workflow_result::Status::Cancelled(_));
     Ok(().into())
 }

package/sdk-core/core/src/core_tests/determinism.rs

@@ -1,4 +1,5 @@
 use crate::{
+    internal_flags::CoreInternalFlags,
     replay::DEFAULT_WORKFLOW_TYPE,
     test_help::{canned_histories, mock_sdk, mock_sdk_cfg, MockPollCfg, ResponseType},
     worker::client::mocks::mock_workflow_client,
@@ -8,8 +9,13 @@ use std::{
     time::Duration,
 };
 use temporal_client::WorkflowOptions;
-use temporal_sdk::{WfContext, WorkflowResult};
-use temporal_sdk_core_protos::temporal::api::enums::v1::WorkflowTaskFailedCause;
+use temporal_sdk::{
+    ActivityOptions, ChildWorkflowOptions, LocalActivityOptions, WfContext, WorkflowResult,
+};
+use temporal_sdk_core_protos::{
+    temporal::api::{enums::v1::WorkflowTaskFailedCause, failure::v1::Failure},
+    DEFAULT_ACTIVITY_TYPE,
+};
 
 static DID_FAIL: AtomicBool = AtomicBool::new(false);
 pub async fn timer_wf_fails_once(ctx: WfContext) -> WorkflowResult<()> {
@@ -105,3 +111,160 @@ async fn test_wf_task_rejected_properly_due_to_nondeterminism(#[case] use_cache:
     // timer and proceed without restarting
     assert_eq!(2, started_count.load(Ordering::Relaxed));
 }
+
+#[rstest::rstest]
+#[tokio::test]
+async fn activity_id_or_type_change_is_nondeterministic(
+    #[values(true, false)] use_cache: bool,
+    #[values(true, false)] id_change: bool,
+    #[values(true, false)] local_act: bool,
+) {
+    let wf_id = "fakeid";
+    let wf_type = DEFAULT_WORKFLOW_TYPE;
+    let mut t = if local_act {
+        canned_histories::single_local_activity("1")
+    } else {
+        canned_histories::single_activity("1")
+    };
+    t.set_flags_first_wft(&[CoreInternalFlags::IdAndTypeDeterminismChecks as u32], &[]);
+    let mock = mock_workflow_client();
+    let mut mh = MockPollCfg::from_resp_batches(
+        wf_id,
+        t,
+        // Two polls are needed, since the first will fail
+        [ResponseType::AllHistory, ResponseType::AllHistory],
+        mock,
+    );
+    // We should see one wft failure which has nondeterminism cause
+    mh.num_expected_fails = 1;
+    mh.expect_fail_wft_matcher = Box::new(move |_, cause, f| {
+        let should_contain = if id_change {
+            "does not match activity id"
+        } else {
+            "does not match activity type"
+        };
+        matches!(cause, WorkflowTaskFailedCause::NonDeterministicError)
+            && matches!(f, Some(Failure {
+                message,
+                ..
+            }) if message.contains(should_contain))
+    });
+    let mut worker = mock_sdk_cfg(mh, |cfg| {
+        if use_cache {
+            cfg.max_cached_workflows = 2;
+        }
+    });
+
+    worker.register_wf(wf_type.to_owned(), move |ctx: WfContext| async move {
+        if local_act {
+            ctx.local_activity(if id_change {
+                LocalActivityOptions {
+                    activity_id: Some("I'm bad and wrong!".to_string()),
+                    activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+                    ..Default::default()
+                }
+            } else {
+                LocalActivityOptions {
+                    activity_type: "not the default act type".to_string(),
+                    ..Default::default()
+                }
+            })
+            .await;
+        } else {
+            ctx.activity(if id_change {
+                ActivityOptions {
+                    activity_id: Some("I'm bad and wrong!".to_string()),
+                    activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+                    ..Default::default()
+                }
+            } else {
+                ActivityOptions {
+                    activity_type: "not the default act type".to_string(),
+                    ..Default::default()
+                }
+            })
+            .await;
+        }
+        Ok(().into())
+    });
+
+    worker
+        .submit_wf(
+            wf_id.to_owned(),
+            wf_type.to_owned(),
+            vec![],
+            WorkflowOptions::default(),
+        )
+        .await
+        .unwrap();
+    worker.run_until_done().await.unwrap();
+}
+
+#[rstest::rstest]
+#[tokio::test]
+async fn child_wf_id_or_type_change_is_nondeterministic(
+    #[values(true, false)] use_cache: bool,
+    #[values(true, false)] id_change: bool,
+) {
+    let wf_id = "fakeid";
+    let wf_type = DEFAULT_WORKFLOW_TYPE;
+    let mut t = canned_histories::single_child_workflow("1");
+    t.set_flags_first_wft(&[CoreInternalFlags::IdAndTypeDeterminismChecks as u32], &[]);
+    let mock = mock_workflow_client();
+    let mut mh = MockPollCfg::from_resp_batches(
+        wf_id,
+        t,
+        // Two polls are needed, since the first will fail
+        [ResponseType::AllHistory, ResponseType::AllHistory],
+        mock,
+    );
+    // We should see one wft failure which has nondeterminism cause
+    mh.num_expected_fails = 1;
+    mh.expect_fail_wft_matcher = Box::new(move |_, cause, f| {
+        let should_contain = if id_change {
+            "does not match child workflow id"
+        } else {
+            "does not match child workflow type"
+        };
+        matches!(cause, WorkflowTaskFailedCause::NonDeterministicError)
+            && matches!(f, Some(Failure {
+                message,
+                ..
+            }) if message.contains(should_contain))
+    });
+    let mut worker = mock_sdk_cfg(mh, |cfg| {
+        if use_cache {
+            cfg.max_cached_workflows = 2;
+        }
+    });
+
+    worker.register_wf(wf_type.to_owned(), move |ctx: WfContext| async move {
+        ctx.child_workflow(if id_change {
+            ChildWorkflowOptions {
+                workflow_id: "I'm bad and wrong!".to_string(),
+                workflow_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+                ..Default::default()
+            }
+        } else {
+            ChildWorkflowOptions {
+                workflow_id: "1".to_string(),
+                workflow_type: "not the child wf type".to_string(),
+                ..Default::default()
+            }
+        })
+        .start(&ctx)
+        .await;
+        Ok(().into())
+    });
+
+    worker
+        .submit_wf(
+            wf_id.to_owned(),
+            wf_type.to_owned(),
+            vec![],
+            WorkflowOptions::default(),
+        )
+        .await
+        .unwrap();
+    worker.run_until_done().await.unwrap();
+}