@temporalio/core-bridge 1.5.2 → 1.6.0

package/sdk-core/tests/{load_tests.rs → heavy_tests.rs}

@@ -2,16 +2,18 @@ use assert_matches::assert_matches;
 use futures::{future::join_all, sink, stream::FuturesUnordered, StreamExt};
 use std::time::{Duration, Instant};
 use temporal_client::{WfClientExt, WorkflowClientTrait, WorkflowOptions};
-use temporal_sdk::{ActContext, ActivityOptions, WfContext};
+use temporal_sdk::{ActContext, ActivityOptions, WfContext, WorkflowResult};
 use temporal_sdk_core_protos::coresdk::{
     activity_result::ActivityExecutionResult, activity_task::activity_task as act_task,
     workflow_commands::ActivityCancellationType, ActivityTaskCompletion, AsJsonPayloadExt,
 };
-use temporal_sdk_core_test_utils::CoreWfStarter;
+use temporal_sdk_core_test_utils::{workflows::la_problem_workflow, CoreWfStarter};
+
+mod fuzzy_workflow;
 
 #[tokio::test]
 async fn activity_load() {
-    const CONCURRENCY: usize = 1000;
+    const CONCURRENCY: usize = 512;
 
     let mut starter = CoreWfStarter::new("activity_load");
     starter
@@ -54,7 +56,7 @@ async fn activity_load() {
     worker.register_wf(wf_type.to_owned(), wf_fn);
     join_all((0..CONCURRENCY).map(|i| {
         let worker = &worker;
-        let wf_id = format!("activity_load_{}", i);
+        let wf_id = format!("activity_load_{i}");
         async move {
             worker
                 .submit_wf(
@@ -153,7 +155,7 @@ async fn workflow_load() {
 
     let mut workflow_handles = vec![];
     for i in 0..num_workflows {
-        let wfid = format!("{}_{}", wf_name, i);
+        let wfid = format!("{wf_name}_{i}");
         let rid = worker
             .submit_wf(
                 wfid.clone(),
@@ -171,7 +173,7 @@ async fn workflow_load() {
             let sends: FuturesUnordered<_> = (0..num_workflows)
                 .map(|i| {
                     client.signal_workflow_execution(
-                        format!("{}_{}", wf_name, i),
+                        format!("{wf_name}_{i}"),
                         "".to_string(),
                         SIGNAME.to_string(),
                         None,
@@ -187,5 +189,110 @@ async fn workflow_load() {
             tokio::time::sleep(Duration::from_secs(2)).await;
         }
     };
-    tokio::select! { r1 = worker.run_until_done() => {r1.unwrap()}, _ = sig_sender => {}};
+    tokio::select! { r1 = worker.run_until_done() => {r1.unwrap()}, _ = sig_sender => {}}
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn evict_while_la_running_no_interference() {
+    let wf_name = "evict_while_la_running_no_interference";
+    let mut starter = CoreWfStarter::new(wf_name);
+    starter.max_local_at(20);
+    starter.max_cached_workflows(20);
+    // Though it doesn't make sense to set wft higher than cached workflows, leaving this commented
+    // introduces more instability that can be useful in the test.
+    // starter.max_wft(20);
+    let mut worker = starter.worker().await;
+
+    worker.register_wf(wf_name.to_owned(), la_problem_workflow);
+    worker.register_activity("delay", |_: ActContext, _: String| async {
+        tokio::time::sleep(Duration::from_secs(15)).await;
+        Ok(())
+    });
+
+    let client = starter.get_client().await;
+    let subfs = FuturesUnordered::new();
+    for i in 1..100 {
+        let wf_id = format!("{wf_name}-{i}");
+        let run_id = worker
+            .submit_wf(
+                &wf_id,
+                wf_name.to_owned(),
+                vec![],
+                WorkflowOptions::default(),
+            )
+            .await
+            .unwrap();
+        let cw = worker.core_worker.clone();
+        let client = client.clone();
+        subfs.push(async move {
+            // Evict the workflow
+            tokio::time::sleep(Duration::from_secs(1)).await;
+            cw.request_workflow_eviction(&run_id);
+            // Wake up workflow by sending signal
+            client
+                .signal_workflow_execution(
+                    wf_id,
+                    run_id.clone(),
+                    "whaatever".to_string(),
+                    None,
+                    None,
+                )
+                .await
+                .unwrap();
+        });
+    }
+    let runf = async {
+        worker.run_until_done().await.unwrap();
+    };
+    tokio::join!(subfs.collect::<Vec<_>>(), runf);
+}
+
+pub async fn many_parallel_timers_longhist(ctx: WfContext) -> WorkflowResult<()> {
+    for _ in 0..120 {
+        let mut futs = vec![];
+        for _ in 0..100 {
+            futs.push(ctx.timer(Duration::from_millis(100)));
+        }
+        join_all(futs).await;
+    }
+    Ok(().into())
+}
+
+#[tokio::test]
+async fn can_paginate_long_history() {
+    let wf_name = "can_paginate_long_history";
+    let mut starter = CoreWfStarter::new(wf_name);
+    // Do not use sticky queues so we are forced to paginate once history gets long
+    starter.max_cached_workflows(0);
+
+    let mut worker = starter.worker().await;
+    worker.register_wf(wf_name.to_owned(), many_parallel_timers_longhist);
+    let run_id = worker
+        .submit_wf(
+            wf_name.to_owned(),
+            wf_name.to_owned(),
+            vec![],
+            WorkflowOptions::default(),
+        )
+        .await
+        .unwrap();
+    let client = starter.get_client().await;
+    tokio::spawn(async move {
+        loop {
+            for _ in 0..10 {
+                client
+                    .signal_workflow_execution(
+                        wf_name.to_owned(),
+                        run_id.clone(),
+                        "sig".to_string(),
+                        None,
+                        None,
+                    )
+                    .await
+                    .unwrap();
+            }
+            tokio::time::sleep(Duration::from_secs(3)).await;
+        }
+    });
+    worker.run_until_done().await.unwrap();
 }

package/sdk-core/tests/integ_tests/heartbeat_tests.rs

@@ -13,7 +13,10 @@ use temporal_sdk_core_protos::{
         workflow_completion::WorkflowActivationCompletion,
         ActivityHeartbeat, ActivityTaskCompletion, AsJsonPayloadExt, IntoCompletion,
     },
-    temporal::api::common::v1::{Payload, RetryPolicy},
+    temporal::api::{
+        common::v1::{Payload, RetryPolicy},
+        enums::v1::TimeoutType,
+    },
 };
 use temporal_sdk_core_test_utils::{
     init_core_and_create_wf, schedule_activity_cmd, CoreWfStarter, WorkerTestHelpers,
@@ -196,7 +199,7 @@ async fn activity_doesnt_heartbeat_hits_timeout_then_completes() {
                 ..Default::default()
             })
             .await;
-        assert!(res.timed_out());
+        assert_eq!(res.timed_out(), Some(TimeoutType::Heartbeat));
         Ok(().into())
     });
 

package/sdk-core/tests/integ_tests/metrics_tests.rs

@@ -22,7 +22,7 @@ async fn prometheus_metrics_exported() {
         .await
         .unwrap();
 
-    let body = reqwest::get(format!("http://{}/metrics", addr))
+    let body = reqwest::get(format!("http://{addr}/metrics"))
         .await
         .unwrap()
         .text()

package/sdk-core/tests/integ_tests/polling_tests.rs

@@ -1,8 +1,5 @@
 use assert_matches::assert_matches;
-use futures::future::join_all;
 use std::time::Duration;
-use temporal_client::WorkflowOptions;
-use temporal_sdk::{WfContext, WorkflowResult};
 use temporal_sdk_core_protos::coresdk::{
     activity_task::activity_task as act_task,
     workflow_activation::{workflow_activation_job, FireTimer, WorkflowActivationJob},
@@ -11,7 +8,7 @@ use temporal_sdk_core_protos::coresdk::{
     IntoCompletion,
 };
 use temporal_sdk_core_test_utils::{
-    init_core_and_create_wf, schedule_activity_cmd, CoreWfStarter, WorkerTestHelpers,
+    init_core_and_create_wf, schedule_activity_cmd, WorkerTestHelpers,
 };
 use tokio::time::timeout;
 
@@ -96,38 +93,3 @@ async fn out_of_order_completion_doesnt_hang() {
 
     jh.await.unwrap();
 }
-
-pub async fn many_parallel_timers_longhist(ctx: WfContext) -> WorkflowResult<()> {
-    for _ in 0..20 {
-        let mut futs = vec![];
-        for _ in 0..1000 {
-            futs.push(ctx.timer(Duration::from_millis(100)));
-        }
-        join_all(futs).await;
-    }
-    Ok(().into())
-}
-
-// Ignored for now because I can't actually get this to produce pages. Need to generate some
-// large payloads I think.
-#[tokio::test]
-#[ignore]
-async fn can_paginate_long_history() {
-    let wf_name = "can_paginate_long_history";
-    let mut starter = CoreWfStarter::new(wf_name);
-    // Do not use sticky queues so we are forced to paginate once history gets long
-    starter.max_cached_workflows(0);
-
-    let mut worker = starter.worker().await;
-    worker.register_wf(wf_name.to_owned(), many_parallel_timers_longhist);
-    worker
-        .submit_wf(
-            wf_name.to_owned(),
-            wf_name.to_owned(),
-            vec![],
-            WorkflowOptions::default(),
-        )
-        .await
-        .unwrap();
-    worker.run_until_done().await.unwrap();
-}

package/sdk-core/tests/integ_tests/queries_tests.rs

@@ -10,7 +10,7 @@ use temporal_sdk_core_protos::{
     },
     temporal::api::{failure::v1::Failure, query::v1::WorkflowQuery},
 };
-use temporal_sdk_core_test_utils::{init_core_and_create_wf, CoreWfStarter, WorkerTestHelpers};
+use temporal_sdk_core_test_utils::{init_core_and_create_wf, WorkerTestHelpers};
 
 #[tokio::test]
 async fn simple_query_legacy() {
@@ -112,7 +112,7 @@ async fn simple_query_legacy() {
 async fn query_after_execution_complete(#[case] do_evict: bool) {
     let query_resp = b"response";
     let mut starter =
-        init_core_and_create_wf(&format!("query_after_execution_complete-{}", do_evict)).await;
+        init_core_and_create_wf(&format!("query_after_execution_complete-{do_evict}")).await;
     let core = &starter.get_worker().await;
     let workflow_id = &starter.get_task_queue().to_string();
 
@@ -208,131 +208,6 @@ async fn query_after_execution_complete(#[case] do_evict: bool) {
     core.shutdown().await;
 }
 
-#[ignore]
-#[tokio::test]
-async fn repros_query_dropped_on_floor() {
-    // This test reliably repros the server dropping one of the two simultaneously issued queries.
-    let q1_resp = b"query_1_resp";
-    let q2_resp = b"query_2_resp";
-    let mut wf_starter = CoreWfStarter::new("repros_query_dropped_on_floor");
-    // Easiest way I discovered to reliably trigger new query path is with a WFT timeout
-    wf_starter.wft_timeout(Duration::from_secs(1));
-    let core = wf_starter.get_worker().await;
-    let task_q = wf_starter.get_task_queue().to_string();
-    wf_starter.start_wf().await;
-    let client = wf_starter.get_client().await;
-
-    let task = core.poll_workflow_activation().await.unwrap();
-    core.complete_timer(&task.run_id, 1, Duration::from_millis(500))
-        .await;
-
-    // Poll for a task we will time out
-    let task = core.poll_workflow_activation().await.unwrap();
-    tokio::time::sleep(Duration::from_secs(2)).await;
-    // Complete now-timed-out task (add a new timer)
-    core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
-        task.run_id.clone(),
-        vec![],
-    ))
-    .await
-    .unwrap();
-
-    let run_id = task.run_id.to_string();
-    let q1_fut = async {
-        client
-            .query_workflow_execution(
-                task_q.clone(),
-                run_id,
-                WorkflowQuery {
-                    query_type: "query_1".to_string(),
-                    query_args: Some(b"hi 1".into()),
-                    header: None,
-                },
-            )
-            .await
-            .unwrap()
-    };
-    let run_id = task.run_id.to_string();
-    let q2_fut = async {
-        client
-            .query_workflow_execution(
-                task_q.clone(),
-                run_id,
-                WorkflowQuery {
-                    query_type: "query_2".to_string(),
-                    query_args: Some(b"hi 2".into()),
-                    header: None,
-                },
-            )
-            .await
-            .unwrap()
-    };
-    let workflow_completions_future = async {
-        let mut seen_q1 = false;
-        let mut seen_q2 = false;
-        while !seen_q1 || !seen_q2 {
-            let task = core.poll_workflow_activation().await.unwrap();
-
-            if matches!(
-                task.jobs[0],
-                WorkflowActivationJob {
-                    variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
-                }
-            ) {
-                let task = core.poll_workflow_activation().await.unwrap();
-                core.complete_timer(&task.run_id, 1, Duration::from_millis(500))
-                    .await;
-                continue;
-            }
-
-            if matches!(
-                task.jobs[0],
-                WorkflowActivationJob {
-                    variant: Some(workflow_activation_job::Variant::FireTimer(_)),
-                }
-            ) {
-                // If we get the timer firing after replay, be done.
-                core.complete_execution(&task.run_id).await;
-            }
-
-            // There should be a query job (really, there should be both... server only sends one?)
-            let query = assert_matches!(
-                task.jobs.as_slice(),
-                [WorkflowActivationJob {
-                    variant: Some(workflow_activation_job::Variant::QueryWorkflow(q)),
-                }] => q
-            );
-            let resp = if query.query_type == "query_1" {
-                seen_q1 = true;
-                q1_resp
-            } else {
-                seen_q2 = true;
-                q2_resp
-            };
-            // Complete the query
-            core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
-                task.run_id,
-                vec![QueryResult {
-                    query_id: query.query_id.clone(),
-                    variant: Some(
-                        QuerySuccess {
-                            response: Some(resp.into()),
-                        }
-                        .into(),
-                    ),
-                }
-                .into()],
-            ))
-            .await
-            .unwrap();
-        }
-    };
-    let (q1_res, q2_res, _) = tokio::join!(q1_fut, q2_fut, workflow_completions_future);
-    // Ensure query responses are as expected
-    assert_eq!(&q1_res.unwrap()[0].data, q1_resp);
-    assert_eq!(&q2_res.unwrap()[0].data, q2_resp);
-}
-
 #[tokio::test]
 async fn fail_legacy_query() {
     let query_err = "oh no broken";

package/sdk-core/tests/integ_tests/visibility_tests.rs

@@ -1,8 +1,8 @@
 use assert_matches::assert_matches;
 use std::{sync::Arc, time::Duration};
 use temporal_client::{
-    ListClosedFilters, ListOpenFilters, Namespace, StartTimeFilter, WorkflowClientTrait,
-    WorkflowExecutionFilter, WorkflowOptions,
+    ListClosedFilters, ListOpenFilters, Namespace, RegisterNamespaceOptions, StartTimeFilter,
+    WorkflowClientTrait, WorkflowExecutionFilter,
 };
 use temporal_sdk_core_protos::coresdk::workflow_activation::{
     workflow_activation_job, WorkflowActivationJob,
@@ -10,6 +10,7 @@ use temporal_sdk_core_protos::coresdk::workflow_activation::{
 use temporal_sdk_core_test_utils::{
     get_integ_server_options, CoreWfStarter, WorkerTestHelpers, NAMESPACE,
 };
+use tokio::time::sleep;
 
 #[tokio::test]
 async fn client_list_open_closed_workflow_executions() {
@@ -22,9 +23,7 @@ async fn client_list_open_closed_workflow_executions() {
     let latest = earliest + Duration::from_secs(60);
 
     // start workflow
-    let run_id = starter
-        .start_wf_with_id(wf_name.to_owned(), WorkflowOptions::default())
-        .await;
+    let run_id = starter.start_wf_with_id(wf_name.to_owned()).await;
     let task = core.poll_workflow_activation().await.unwrap();
     assert_matches!(
         task.jobs.as_slice(),
@@ -52,6 +51,7 @@ async fn client_list_open_closed_workflow_executions() {
 
     // Complete workflow
     core.complete_execution(&task.run_id).await;
+    core.shutdown().await;
 
     // List above CLOSED workflow
     let start_time_filter = StartTimeFilter {
@@ -76,6 +76,53 @@ async fn client_list_open_closed_workflow_executions() {
     assert_eq!(workflow.execution.as_ref().unwrap().workflow_id, wf_name);
 }
 
+#[tokio::test]
+async fn client_create_namespace() {
+    let client = Arc::new(
+        get_integ_server_options()
+            .connect(NAMESPACE.to_owned(), None, None)
+            .await
+            .expect("Must connect"),
+    );
+
+    let register_options = RegisterNamespaceOptions::builder()
+        .namespace("test-create-namespace")
+        .description("it's alive")
+        .build()
+        .unwrap();
+
+    client
+        .register_namespace(register_options.clone())
+        .await
+        .unwrap();
+
+    //#Hack, not sure how else to wait for a proper response.  RegisterNamespace isn't safe to read
+    //after write
+    let mut attempts = 0;
+    let wait_time = Duration::from_secs(1);
+    loop {
+        attempts += 1;
+        let resp = client
+            .describe_namespace(Namespace::Name(register_options.namespace.clone()))
+            .await;
+
+        match resp {
+            Ok(n) => {
+                let namespace_info = n.namespace_info.unwrap();
+                assert_eq!(namespace_info.name, register_options.namespace);
+                assert_eq!(namespace_info.description, register_options.description);
+                return;
+            }
+            _ => {
+                if attempts == 12 {
+                    panic!("failed to query registered namespace");
+                }
+                sleep(wait_time).await
+            }
+        }
+    }
+}
+
 #[tokio::test]
 async fn client_describe_namespace() {
     let client = Arc::new(

package/sdk-core/tests/integ_tests/workflow_tests/activities.rs

@@ -1,3 +1,4 @@
+use anyhow::anyhow;
 use assert_matches::assert_matches;
 use std::time::Duration;
 use temporal_client::{WfClientExt, WorkflowClientTrait, WorkflowExecutionResult, WorkflowOptions};
@@ -205,6 +206,78 @@ async fn activity_non_retryable_failure() {
     core.complete_execution(&task.run_id).await;
 }
 
+#[tokio::test]
+async fn activity_non_retryable_failure_with_error() {
+    let mut starter = init_core_and_create_wf("activity_non_retryable_failure").await;
+    let core = starter.get_worker().await;
+    let task_q = starter.get_task_queue();
+    let activity_id = "act-1";
+    let task = core.poll_workflow_activation().await.unwrap();
+    // Complete workflow task and schedule activity
+    core.complete_workflow_activation(
+        schedule_activity_cmd(
+            0,
+            task_q,
+            activity_id,
+            ActivityCancellationType::TryCancel,
+            Duration::from_secs(60),
+            Duration::from_secs(60),
+        )
+        .into_completion(task.run_id),
+    )
+    .await
+    .unwrap();
+    // Poll activity and verify that it's been scheduled with correct parameters
+    let task = core.poll_activity_task().await.unwrap();
+    assert_matches!(
+        task.variant,
+        Some(act_task::Variant::Start(start_activity)) => {
+            assert_eq!(start_activity.activity_type, "test_activity".to_string())
+        }
+    );
+    // Fail activity with non-retryable error
+    let failure = Failure::application_failure_from_error(anyhow!("activity failed"), true);
+    core.complete_activity_task(ActivityTaskCompletion {
+        task_token: task.task_token,
+        result: Some(ActivityExecutionResult::fail(failure.clone())),
+    })
+    .await
+    .unwrap();
+    // Poll workflow task and verify that activity has failed.
+    let task = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        task.jobs.as_slice(),
+        [
+            WorkflowActivationJob {
+                variant: Some(workflow_activation_job::Variant::ResolveActivity(
+                    ResolveActivity {seq, result: Some(ActivityResolution{
+                    status: Some(act_res::Status::Failed(activity_result::Failure{
+                        failure: Some(f),
+                    }))})}
+                )),
+            },
+        ] => {
+            assert_eq!(*seq, 0);
+            assert_eq!(f, &Failure{
+                message: "Activity task failed".to_owned(),
+                cause: Some(Box::new(failure)),
+                failure_info: Some(FailureInfo::ActivityFailureInfo(ActivityFailureInfo{
+                    activity_id: "act-1".to_owned(),
+                    activity_type: Some(ActivityType {
+                        name: "test_activity".to_owned(),
+                    }),
+                    scheduled_event_id: 5,
+                    started_event_id: 6,
+                    identity: "integ_tester".to_owned(),
+                    retry_state: RetryState::NonRetryableFailure as i32,
+                })),
+                ..Default::default()
+            });
+        }
+    );
+    core.complete_execution(&task.run_id).await;
+}
+
 #[tokio::test]
 async fn activity_retry() {
     let mut starter = init_core_and_create_wf("activity_retry").await;
@@ -822,7 +895,7 @@ async fn it_can_complete_async() {
             Some(act_res::Status::Completed(activity_result::Success { result })) => result
                 .map(|p| String::from_json_payload(&p).unwrap())
                 .unwrap(),
-            _ => panic!("activity task failed {:?}", activity_resolution),
+            _ => panic!("activity task failed {activity_resolution:?}"),
         };
 
         assert_eq!(&res, async_response);

package/sdk-core/tests/integ_tests/workflow_tests/cancel_wf.rs

@@ -1,5 +1,5 @@
 use std::time::Duration;
-use temporal_client::{WorkflowClientTrait, WorkflowOptions};
+use temporal_client::WorkflowClientTrait;
 use temporal_sdk::{WfContext, WfExitValue, WorkflowResult};
 use temporal_sdk_core_protos::temporal::api::enums::v1::WorkflowExecutionStatus;
 use temporal_sdk_core_test_utils::CoreWfStarter;
@@ -24,22 +24,14 @@ async fn cancel_during_timer() {
     let mut worker = starter.worker().await;
     let client = starter.get_client().await;
     worker.register_wf(wf_name.to_string(), cancelled_wf);
-
-    worker
-        .submit_wf(
-            wf_name.to_owned(),
-            wf_name.to_owned(),
-            vec![],
-            WorkflowOptions::default(),
-        )
-        .await
-        .unwrap();
+    starter.start_with_worker(wf_name, &mut worker).await;
+    let wf_id = starter.get_task_queue().to_string();
 
     let canceller = async {
         tokio::time::sleep(Duration::from_millis(500)).await;
         // Cancel the workflow externally
         client
-            .cancel_workflow_execution(wf_name.to_string(), None, "Dieee".to_string(), None)
+            .cancel_workflow_execution(wf_id.clone(), None, "Dieee".to_string(), None)
             .await
             .unwrap();
     };
@@ -47,7 +39,7 @@ async fn cancel_during_timer() {
     let (_, res) = tokio::join!(canceller, worker.run_until_done());
     res.unwrap();
     let desc = client
-        .describe_workflow_execution(wf_name.to_string(), None)
+        .describe_workflow_execution(wf_id, None)
         .await
         .unwrap();
 

package/sdk-core/tests/integ_tests/workflow_tests/continue_as_new.rs

@@ -44,7 +44,7 @@ async fn continue_as_new_multiple_concurrent() {
     let mut worker = starter.worker().await;
     worker.register_wf(wf_name.to_string(), continue_as_new_wf);
 
-    let wf_names = (1..=20).map(|i| format!("{}-{}", wf_name, i));
+    let wf_names = (1..=20).map(|i| format!("{wf_name}-{i}"));
     for name in wf_names.clone() {
         worker
             .submit_wf(

package/sdk-core/tests/integ_tests/workflow_tests/determinism.rs

@@ -2,7 +2,7 @@ use std::{
     sync::atomic::{AtomicUsize, Ordering},
     time::Duration,
 };
-use temporal_client::WorkflowOptions;
+
 use temporal_sdk::{ActivityOptions, WfContext, WorkflowResult};
 use temporal_sdk_core_test_utils::CoreWfStarter;
 
@@ -39,15 +39,7 @@ async fn test_determinism_error_then_recovers() {
     let mut worker = starter.worker().await;
 
     worker.register_wf(wf_name.to_owned(), timer_wf_nondeterministic);
-    worker
-        .submit_wf(
-            wf_name.to_owned(),
-            wf_name.to_owned(),
-            vec![],
-            WorkflowOptions::default(),
-        )
-        .await
-        .unwrap();
+    starter.start_with_worker(wf_name, &mut worker).await;
     worker.run_until_done().await.unwrap();
     // 4 because we still add on the 3rd and final attempt
     assert_eq!(RUN_CT.load(Ordering::Relaxed), 4);