@temporalio/core-bridge 1.5.2 → 1.7.0

package/sdk-core/tests/integ_tests/workflow_tests/determinism.rs

@@ -2,7 +2,7 @@ use std::{
     sync::atomic::{AtomicUsize, Ordering},
     time::Duration,
 };
-use temporal_client::WorkflowOptions;
+
 use temporal_sdk::{ActivityOptions, WfContext, WorkflowResult};
 use temporal_sdk_core_test_utils::CoreWfStarter;
 
@@ -36,18 +36,11 @@ pub async fn timer_wf_nondeterministic(ctx: WfContext) -> WorkflowResult<()> {
 async fn test_determinism_error_then_recovers() {
     let wf_name = "test_determinism_error_then_recovers";
     let mut starter = CoreWfStarter::new(wf_name);
+    starter.no_remote_activities();
     let mut worker = starter.worker().await;
 
     worker.register_wf(wf_name.to_owned(), timer_wf_nondeterministic);
-    worker
-        .submit_wf(
-            wf_name.to_owned(),
-            wf_name.to_owned(),
-            vec![],
-            WorkflowOptions::default(),
-        )
-        .await
-        .unwrap();
+    starter.start_with_worker(wf_name, &mut worker).await;
     worker.run_until_done().await.unwrap();
     // 4 because we still add on the 3rd and final attempt
     assert_eq!(RUN_CT.load(Ordering::Relaxed), 4);

package/sdk-core/tests/integ_tests/workflow_tests/local_activities.rs

@@ -1,23 +1,27 @@
 use anyhow::anyhow;
 use futures::future::join_all;
-use futures_util::stream::{FuturesUnordered, StreamExt};
-use std::time::Duration;
-use temporal_client::{WorkflowClientTrait, WorkflowOptions};
+use std::{
+    sync::atomic::{AtomicU8, Ordering},
+    time::Duration,
+};
+use temporal_client::WorkflowOptions;
 use temporal_sdk::{
-    interceptors::WorkerInterceptor, ActContext, ActivityCancelledError, ActivityOptions,
-    CancellableFuture, LocalActivityOptions, WfContext, WorkflowResult,
+    interceptors::WorkerInterceptor, ActContext, ActivityCancelledError, CancellableFuture,
+    LocalActivityOptions, WfContext, WorkflowResult,
 };
 use temporal_sdk_core::replay::HistoryForReplay;
 use temporal_sdk_core_protos::{
     coresdk::{
-        workflow_commands::ActivityCancellationType,
+        workflow_commands::workflow_command::Variant, workflow_commands::ActivityCancellationType,
+        workflow_completion, workflow_completion::workflow_activation_completion,
         workflow_completion::WorkflowActivationCompletion, AsJsonPayloadExt,
     },
-    temporal::api::common::v1::RetryPolicy,
+    temporal::api::{common::v1::RetryPolicy, enums::v1::TimeoutType},
     TestHistoryBuilder,
 };
 use temporal_sdk_core_test_utils::{
-    history_from_proto_binary, init_integ_telem, replay_sdk_worker, CoreWfStarter,
+    history_from_proto_binary, init_integ_telem, replay_sdk_worker, workflows::la_problem_workflow,
+    CoreWfStarter,
 };
 use tokio_util::sync::CancellationToken;
 
@@ -46,15 +50,7 @@ async fn one_local_activity() {
     worker.register_wf(wf_name.to_owned(), one_local_activity_wf);
     worker.register_activity("echo_activity", echo);
 
-    worker
-        .submit_wf(
-            wf_name.to_owned(),
-            wf_name.to_owned(),
-            vec![],
-            WorkflowOptions::default(),
-        )
-        .await
-        .unwrap();
+    starter.start_with_worker(wf_name, &mut worker).await;
     worker.run_until_done().await.unwrap();
 }
 
@@ -77,15 +73,7 @@ async fn local_act_concurrent_with_timer() {
     worker.register_wf(wf_name.to_owned(), local_act_concurrent_with_timer_wf);
     worker.register_activity("echo_activity", echo);
 
-    worker
-        .submit_wf(
-            wf_name.to_owned(),
-            wf_name.to_owned(),
-            vec![],
-            WorkflowOptions::default(),
-        )
-        .await
-        .unwrap();
+    starter.start_with_worker(wf_name, &mut worker).await;
     worker.run_until_done().await.unwrap();
 }
 
@@ -109,15 +97,7 @@ async fn local_act_then_timer_then_wait_result() {
     worker.register_wf(wf_name.to_owned(), local_act_then_timer_then_wait);
     worker.register_activity("echo_activity", echo);
 
-    worker
-        .submit_wf(
-            wf_name.to_owned(),
-            wf_name.to_owned(),
-            vec![],
-            WorkflowOptions::default(),
-        )
-        .await
-        .unwrap();
+    starter.start_with_worker(wf_name, &mut worker).await;
     worker.run_until_done().await.unwrap();
 }
 
@@ -125,7 +105,7 @@ async fn local_act_then_timer_then_wait_result() {
 async fn long_running_local_act_with_timer() {
     let wf_name = "long_running_local_act_with_timer";
     let mut starter = CoreWfStarter::new(wf_name);
-    starter.wft_timeout(Duration::from_secs(1));
+    starter.workflow_options.task_timeout = Some(Duration::from_secs(1));
     let mut worker = starter.worker().await;
     worker.register_wf(wf_name.to_owned(), local_act_then_timer_then_wait);
     worker.register_activity("echo_activity", |_ctx: ActContext, str: String| async {
@@ -133,15 +113,7 @@ async fn long_running_local_act_with_timer() {
         Ok(str)
     });
 
-    worker
-        .submit_wf(
-            wf_name.to_owned(),
-            wf_name.to_owned(),
-            vec![],
-            WorkflowOptions::default(),
-        )
-        .await
-        .unwrap();
+    starter.start_with_worker(wf_name, &mut worker).await;
     worker.run_until_done().await.unwrap();
 }
 
@@ -150,7 +122,7 @@ pub async fn local_act_fanout_wf(ctx: WfContext) -> WorkflowResult<()> {
         .map(|i| {
             ctx.local_activity(LocalActivityOptions {
                 activity_type: "echo_activity".to_string(),
-                input: format!("Hi {}", i)
+                input: format!("Hi {i}")
                     .as_json_payload()
                     .expect("serializes fine"),
                 ..Default::default()
@@ -171,15 +143,7 @@ async fn local_act_fanout() {
     worker.register_wf(wf_name.to_owned(), local_act_fanout_wf);
     worker.register_activity("echo_activity", echo);
 
-    worker
-        .submit_wf(
-            wf_name.to_owned(),
-            wf_name.to_owned(),
-            vec![],
-            WorkflowOptions::default(),
-        )
-        .await
-        .unwrap();
+    starter.start_with_worker(wf_name, &mut worker).await;
     worker.run_until_done().await.unwrap();
 }
 
@@ -234,7 +198,7 @@ async fn local_act_retry_timer_backoff() {
 #[case::abandon(ActivityCancellationType::Abandon)]
 #[tokio::test]
 async fn cancel_immediate(#[case] cancel_type: ActivityCancellationType) {
-    let wf_name = format!("cancel_immediate_{:?}", cancel_type);
+    let wf_name = format!("cancel_immediate_{cancel_type:?}");
     let mut starter = CoreWfStarter::new(&wf_name);
     let mut worker = starter.worker().await;
     worker.register_wf(&wf_name, move |ctx: WfContext| async move {
@@ -268,17 +232,10 @@ async fn cancel_immediate(#[case] cancel_type: ActivityCancellationType) {
         }
     });
 
-    worker
-        .submit_wf(
-            wf_name.to_owned(),
-            wf_name.to_owned(),
-            vec![],
-            WorkflowOptions::default(),
-        )
-        .await
-        .unwrap();
+    starter.start_with_worker(wf_name, &mut worker).await;
     worker
         .run_until_done_intercepted(Some(LACancellerInterceptor {
+            cancel_on_workflow_completed: false,
             token: manual_cancel,
         }))
         .await
@@ -287,12 +244,29 @@ async fn cancel_immediate(#[case] cancel_type: ActivityCancellationType) {
 
 struct LACancellerInterceptor {
     token: CancellationToken,
+    cancel_on_workflow_completed: bool,
 }
 #[async_trait::async_trait(?Send)]
 impl WorkerInterceptor for LACancellerInterceptor {
-    async fn on_workflow_activation_completion(&self, _: &WorkflowActivationCompletion) {}
+    async fn on_workflow_activation_completion(&self, completion: &WorkflowActivationCompletion) {
+        if !self.cancel_on_workflow_completed {
+            return;
+        }
+        if let Some(workflow_activation_completion::Status::Successful(
+            workflow_completion::Success { commands, .. },
+        )) = completion.status.as_ref()
+        {
+            if let Some(&Variant::CompleteWorkflowExecution(_)) =
+                commands.last().and_then(|v| v.variant.as_ref())
+            {
+                self.token.cancel();
+            }
+        }
+    }
     fn on_shutdown(&self, _: &temporal_sdk::Worker) {
-        self.token.cancel()
+        if !self.cancel_on_workflow_completed {
+            self.token.cancel()
+        }
     }
 }
 
@@ -310,12 +284,9 @@ async fn cancel_after_act_starts(
     )]
     cancel_type: ActivityCancellationType,
 ) {
-    let wf_name = format!(
-        "cancel_after_act_starts_timer_{:?}_{:?}",
-        cancel_on_backoff, cancel_type
-    );
+    let wf_name = format!("cancel_after_act_starts_{cancel_on_backoff:?}_{cancel_type:?}");
     let mut starter = CoreWfStarter::new(&wf_name);
-    starter.wft_timeout(Duration::from_secs(1));
+    starter.workflow_options.task_timeout = Some(Duration::from_secs(1));
     let mut worker = starter.worker().await;
     let bo_dur = cancel_on_backoff.unwrap_or_else(|| Duration::from_secs(1));
     worker.register_wf(&wf_name, move |ctx: WfContext| async move {
@@ -375,21 +346,17 @@ async fn cancel_after_act_starts(
         }
     });
 
-    worker
-        .submit_wf(
-            wf_name.to_owned(),
-            wf_name.to_owned(),
-            vec![],
-            WorkflowOptions::default(),
-        )
-        .await
-        .unwrap();
+    starter.start_with_worker(&wf_name, &mut worker).await;
     worker
         .run_until_done_intercepted(Some(LACancellerInterceptor {
             token: manual_cancel,
+            // Only needed for this one case since the activity is not drained and prevents worker from shutting down.
+            cancel_on_workflow_completed: matches!(cancel_type, ActivityCancellationType::Abandon)
+                && cancel_on_backoff.is_none(),
         }))
         .await
         .unwrap();
+    starter.shutdown().await;
 }
 
 #[rstest::rstest]
@@ -408,6 +375,11 @@ async fn x_to_close_timeout(#[case] is_schedule: bool) {
     } else {
         (None, Some(Duration::from_secs(2)))
     };
+    let timeout_type = if is_schedule {
+        TimeoutType::ScheduleToClose
+    } else {
+        TimeoutType::StartToClose
+    };
 
     worker.register_wf(wf_name.to_owned(), move |ctx: WfContext| async move {
         let res = ctx
@@ -427,7 +399,7 @@ async fn x_to_close_timeout(#[case] is_schedule: bool) {
                 ..Default::default()
             })
             .await;
-        assert!(res.timed_out());
+        assert_eq!(res.timed_out(), Some(timeout_type));
         Ok(().into())
     });
     worker.register_activity("echo", |ctx: ActContext, _: String| async move {
@@ -440,15 +412,7 @@ async fn x_to_close_timeout(#[case] is_schedule: bool) {
         Ok(())
     });
 
-    worker
-        .submit_wf(
-            wf_name.to_owned(),
-            wf_name.to_owned(),
-            vec![],
-            WorkflowOptions::default(),
-        )
-        .await
-        .unwrap();
+    starter.start_with_worker(wf_name, &mut worker).await;
     worker.run_until_done().await.unwrap();
 }
 
@@ -472,43 +436,37 @@ async fn schedule_to_close_timeout_across_timer_backoff(#[case] cached: bool) {
                 activity_type: "echo".to_string(),
                 input: "hi".as_json_payload().expect("serializes fine"),
                 retry_policy: RetryPolicy {
-                    initial_interval: Some(prost_dur!(from_micros(15))),
+                    initial_interval: Some(prost_dur!(from_millis(15))),
                     backoff_coefficient: 1_000.,
-                    maximum_interval: Some(prost_dur!(from_millis(1500))),
+                    maximum_interval: Some(prost_dur!(from_millis(1000))),
                     maximum_attempts: 40,
                     non_retryable_error_types: vec![],
                 },
-                timer_backoff_threshold: Some(Duration::from_secs(1)),
-                schedule_to_close_timeout: Some(Duration::from_secs(3)),
+                timer_backoff_threshold: Some(Duration::from_millis(500)),
+                schedule_to_close_timeout: Some(Duration::from_secs(2)),
                 ..Default::default()
             })
             .await;
-        assert!(res.timed_out());
+        assert_eq!(res.timed_out(), Some(TimeoutType::ScheduleToClose));
         Ok(().into())
     });
-    worker.register_activity("echo", |_: ActContext, _: String| async {
+    let num_attempts: &'static _ = Box::leak(Box::new(AtomicU8::new(0)));
+    worker.register_activity("echo", move |_: ActContext, _: String| async {
+        num_attempts.fetch_add(1, Ordering::Relaxed);
         Result::<(), _>::Err(anyhow!("Oh no I failed!"))
     });
 
-    worker
-        .submit_wf(
-            wf_name.to_owned(),
-            wf_name.to_owned(),
-            vec![],
-            WorkflowOptions::default(),
-        )
-        .await
-        .unwrap();
+    starter.start_with_worker(wf_name, &mut worker).await;
     worker.run_until_done().await.unwrap();
+    // 3 attempts b/c first backoff is very small, then the next 2 attempts take at least 2 seconds
+    // b/c of timer backoff.
+    assert_eq!(3, num_attempts.load(Ordering::Relaxed));
 }
 
 #[rstest::rstest]
 #[tokio::test]
 async fn eviction_wont_make_local_act_get_dropped(#[values(true, false)] short_wft_timeout: bool) {
-    let wf_name = format!(
-        "eviction_wont_make_local_act_get_dropped_{}",
-        short_wft_timeout
-    );
+    let wf_name = format!("eviction_wont_make_local_act_get_dropped_{short_wft_timeout}");
     let mut starter = CoreWfStarter::new(&wf_name);
     starter.max_cached_workflows(0);
     let mut worker = starter.worker().await;
@@ -574,15 +532,7 @@ async fn timer_backoff_concurrent_with_non_timer_backoff() {
         Result::<(), _>::Err(anyhow!("Oh no I failed!"))
     });
 
-    worker
-        .submit_wf(
-            wf_name.to_owned(),
-            wf_name.to_owned(),
-            vec![],
-            WorkflowOptions::default(),
-        )
-        .await
-        .unwrap();
+    starter.start_with_worker(wf_name, &mut worker).await;
     worker.run_until_done().await.unwrap();
 }
 
@@ -613,7 +563,7 @@ async fn repro_nondeterminism_with_timer_bug() {
             _ = r1 => {
                 t1.cancel(&ctx);
             },
-        };
+        }
         ctx.timer(Duration::from_secs(1)).await;
         Ok(().into())
     });
@@ -638,85 +588,6 @@ async fn repro_nondeterminism_with_timer_bug() {
         .unwrap();
 }
 
-async fn la_problem_workflow(ctx: WfContext) -> WorkflowResult<()> {
-    ctx.local_activity(LocalActivityOptions {
-        activity_type: "delay".to_string(),
-        input: "hi".as_json_payload().expect("serializes fine"),
-        retry_policy: RetryPolicy {
-            initial_interval: Some(prost_dur!(from_micros(15))),
-            backoff_coefficient: 1_000.,
-            maximum_interval: Some(prost_dur!(from_millis(1500))),
-            maximum_attempts: 4,
-            non_retryable_error_types: vec![],
-        },
-        timer_backoff_threshold: Some(Duration::from_secs(1)),
-        ..Default::default()
-    })
-    .await;
-    ctx.activity(ActivityOptions {
-        activity_type: "delay".to_string(),
-        start_to_close_timeout: Some(Duration::from_secs(20)),
-        input: "hi!".as_json_payload().expect("serializes fine"),
-        ..Default::default()
-    })
-    .await;
-    Ok(().into())
-}
-
-// Expensive to run - worth enabling on a stress/regression pipeline.
-#[ignore]
-#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-async fn evict_while_la_running_no_interference() {
-    let wf_name = "evict_while_la_running_no_interference";
-    let mut starter = CoreWfStarter::new(wf_name);
-    starter.max_local_at(20);
-    starter.max_cached_workflows(20);
-    let mut worker = starter.worker().await;
-
-    worker.register_wf(wf_name.to_owned(), la_problem_workflow);
-    worker.register_activity("delay", |_: ActContext, _: String| async {
-        tokio::time::sleep(Duration::from_secs(15)).await;
-        Ok(())
-    });
-
-    let client = starter.get_client().await;
-    let subfs = FuturesUnordered::new();
-    for i in 1..100 {
-        let wf_id = format!("{}-{}", wf_name, i);
-        let run_id = worker
-            .submit_wf(
-                &wf_id,
-                wf_name.to_owned(),
-                vec![],
-                WorkflowOptions::default(),
-            )
-            .await
-            .unwrap();
-        let cw = worker.core_worker.clone();
-        let client = client.clone();
-        subfs.push(async move {
-            // Evict the workflow
-            tokio::time::sleep(Duration::from_secs(1)).await;
-            cw.request_workflow_eviction(&run_id);
-            // Wake up workflow by sending signal
-            client
-                .signal_workflow_execution(
-                    wf_id,
-                    run_id.clone(),
-                    "whaatever".to_string(),
-                    None,
-                    None,
-                )
-                .await
-                .unwrap();
-        });
-    }
-    let runf = async {
-        worker.run_until_done().await.unwrap();
-    };
-    tokio::join!(subfs.collect::<Vec<_>>(), runf);
-}
-
 #[rstest::rstest]
 #[tokio::test]
 async fn weird_la_nondeterminism_repro(#[values(true, false)] fix_hist: bool) {
@@ -758,7 +629,6 @@ async fn second_weird_la_nondeterminism_repro() {
     // Chop off uninteresting ending
     hist.events.truncate(24);
     let mut thb = TestHistoryBuilder::from_history(hist.events);
-    // thb.add_workflow_task_completed();
     thb.add_workflow_execution_completed();
     hist = thb.get_full_history_info().unwrap().into();
 
@@ -773,3 +643,27 @@ async fn second_weird_la_nondeterminism_repro() {
     });
     worker.run().await.unwrap();
 }
+
+#[tokio::test]
+async fn third_weird_la_nondeterminism_repro() {
+    init_integ_telem();
+    let mut hist = history_from_proto_binary(
+        "histories/evict_while_la_running_no_interference-16_history.bin",
+    )
+    .await
+    .unwrap();
+    let mut thb = TestHistoryBuilder::from_history(hist.events);
+    thb.add_workflow_task_scheduled_and_started();
+    hist = thb.get_full_history_info().unwrap().into();
+
+    let mut worker = replay_sdk_worker([HistoryForReplay::new(hist, "fake".to_owned())]);
+    worker.register_wf(
+        "evict_while_la_running_no_interference",
+        la_problem_workflow,
+    );
+    worker.register_activity("delay", |_: ActContext, _: String| async {
+        tokio::time::sleep(Duration::from_secs(15)).await;
+        Ok(())
+    });
+    worker.run().await.unwrap();
+}

package/sdk-core/tests/integ_tests/workflow_tests/modify_wf_properties.rs

@@ -20,6 +20,7 @@ async fn sends_modify_wf_props() {
     let wf_name = "can_upsert_memo";
     let wf_id = Uuid::new_v4();
     let mut starter = CoreWfStarter::new(wf_name);
+    starter.no_remote_activities();
     let mut worker = starter.worker().await;
 
     worker.register_wf(wf_name, memo_upserter);
@@ -43,10 +44,7 @@ async fn sends_modify_wf_props() {
     let catname = memo.get(FIELD_A).unwrap();
     let cuteness = memo.get(FIELD_B).unwrap();
     for payload in [catname, cuteness] {
-        assert_eq!(
-            &b"json/plain".to_vec(),
-            payload.metadata.get("encoding").unwrap()
-        );
+        assert!(payload.is_json_payload());
     }
     assert_eq!("enchi", String::from_json_payload(catname).unwrap());
     assert_eq!(9001, usize::from_json_payload(cuteness).unwrap());

package/sdk-core/tests/integ_tests/workflow_tests/patches.rs

@@ -2,7 +2,7 @@ use std::{
     sync::atomic::{AtomicBool, Ordering},
     time::Duration,
 };
-use temporal_client::WorkflowOptions;
+
 use temporal_sdk::{WfContext, WorkflowResult};
 use temporal_sdk_core_test_utils::CoreWfStarter;
 
@@ -27,18 +27,11 @@ pub async fn changes_wf(ctx: WfContext) -> WorkflowResult<()> {
 async fn writes_change_markers() {
     let wf_name = "writes_change_markers";
     let mut starter = CoreWfStarter::new(wf_name);
+    starter.no_remote_activities();
     let mut worker = starter.worker().await;
     worker.register_wf(wf_name.to_owned(), changes_wf);
 
-    worker
-        .submit_wf(
-            wf_name.to_owned(),
-            wf_name.to_owned(),
-            vec![],
-            WorkflowOptions::default(),
-        )
-        .await
-        .unwrap();
+    starter.start_with_worker(wf_name, &mut worker).await;
     worker.run_until_done().await.unwrap();
 }
 
@@ -67,18 +60,11 @@ pub async fn no_change_then_change_wf(ctx: WfContext) -> WorkflowResult<()> {
 async fn can_add_change_markers() {
     let wf_name = "can_add_change_markers";
     let mut starter = CoreWfStarter::new(wf_name);
+    starter.no_remote_activities();
     let mut worker = starter.worker().await;
     worker.register_wf(wf_name.to_owned(), no_change_then_change_wf);
 
-    worker
-        .submit_wf(
-            wf_name.to_owned(),
-            wf_name.to_owned(),
-            vec![],
-            WorkflowOptions::default(),
-        )
-        .await
-        .unwrap();
+    starter.start_with_worker(wf_name, &mut worker).await;
     worker.run_until_done().await.unwrap();
 }
 
@@ -97,17 +83,37 @@ pub async fn replay_with_change_marker_wf(ctx: WfContext) -> WorkflowResult<()>
 async fn replaying_with_patch_marker() {
     let wf_name = "replaying_with_patch_marker";
     let mut starter = CoreWfStarter::new(wf_name);
+    starter.no_remote_activities();
     let mut worker = starter.worker().await;
     worker.register_wf(wf_name.to_owned(), replay_with_change_marker_wf);
 
-    worker
-        .submit_wf(
-            wf_name.to_owned(),
-            wf_name.to_owned(),
-            vec![],
-            WorkflowOptions::default(),
-        )
-        .await
-        .unwrap();
+    starter.start_with_worker(wf_name, &mut worker).await;
+    worker.run_until_done().await.unwrap();
+}
+
+/// Test that the internal patching mechanism works on the second workflow task when replaying.
+/// Used as regression test for a bug that detected that we did not look ahead far enough to find
+/// the next workflow task completion, which the flags are attached to.
+#[tokio::test]
+async fn patched_on_second_workflow_task_is_deterministic() {
+    let wf_name = "timer_patched_timer";
+    let mut starter = CoreWfStarter::new(wf_name);
+    // Disable caching to force replay from beginning
+    starter.max_cached_workflows(0).no_remote_activities();
+    let mut worker = starter.worker().await;
+    // Include a task failure as well to make sure that works
+    static FAIL_ONCE: AtomicBool = AtomicBool::new(true);
+    worker.register_wf(wf_name.to_owned(), |ctx: WfContext| async move {
+        ctx.timer(Duration::from_millis(1)).await;
+        if FAIL_ONCE.load(Ordering::Acquire) {
+            FAIL_ONCE.store(false, Ordering::Release);
+            panic!("Enchi is hungry!");
+        }
+        assert!(ctx.patched(MY_PATCH_ID));
+        ctx.timer(Duration::from_millis(1)).await;
+        Ok(().into())
+    });
+
+    starter.start_with_worker(wf_name, &mut worker).await;
     worker.run_until_done().await.unwrap();
 }