@temporalio/core-bridge 1.13.0 → 1.13.2

package/sdk-core/tests/integ_tests/polling_tests.rs

@@ -1,12 +1,34 @@
-use crate::integ_tests::activity_functions::echo;
+use crate::{
+    common::{
+        CoreWfStarter, INTEG_CLIENT_NAME, INTEG_CLIENT_VERSION, get_integ_server_options,
+        init_core_and_create_wf, init_integ_telem, integ_dev_server_config, integ_worker_config,
+    },
+    integ_tests::activity_functions::echo,
+};
 use assert_matches::assert_matches;
-use std::{sync::Arc, time::Duration};
+use futures_util::{FutureExt, StreamExt, future::join_all};
+use std::{
+    process::Stdio,
+    sync::{
+        Arc, Mutex,
+        atomic::{AtomicBool, Ordering},
+    },
+    time::Duration,
+};
 use temporal_client::{WfClientExt, WorkflowClientTrait, WorkflowOptions};
 use temporal_sdk::{ActivityOptions, WfContext};
 use temporal_sdk_core::{
-    ClientOptionsBuilder, ephemeral_server::TemporalDevServerConfigBuilder, init_worker,
+    ClientOptionsBuilder, CoreRuntime,
+    ephemeral_server::{TemporalDevServerConfigBuilder, default_cached_download},
+    init_worker,
+    telemetry::CoreLogStreamConsumer,
+    test_help::{NAMESPACE, WorkerTestHelpers, drain_pollers_and_shutdown},
+};
+use temporal_sdk_core_api::{
+    Worker,
+    telemetry::{Logger, TelemetryOptionsBuilder},
+    worker::PollerBehavior,
 };
-use temporal_sdk_core_api::{Worker, worker::PollerBehavior};
 use temporal_sdk_core_protos::{
     coresdk::{
         AsJsonPayloadExt, IntoCompletion,
@@ -15,13 +37,11 @@ use temporal_sdk_core_protos::{
         workflow_commands::{ActivityCancellationType, RequestCancelActivity, StartTimer},
         workflow_completion::WorkflowActivationCompletion,
     },
+    prost_dur,
     temporal::api::enums::v1::EventType,
+    test_utils::schedule_activity_cmd,
 };
-use temporal_sdk_core_test_utils::{
-    CoreWfStarter, WorkerTestHelpers, default_cached_download, drain_pollers_and_shutdown,
-    init_core_and_create_wf, init_integ_telem, integ_worker_config, schedule_activity_cmd,
-};
-use tokio::time::timeout;
+use tokio::{sync::Notify, time::timeout};
 use tracing::info;
 use url::Url;
 
@@ -115,101 +135,117 @@ async fn switching_worker_client_changes_poll() {
         ])
         .build()
         .unwrap();
-    let mut server1 = server_config.start_server().await.unwrap();
-    let mut server2 = server_config.start_server().await.unwrap();
-
-    // Connect clients to both servers
-    info!("Connecting clients");
-    let mut client_common_config = ClientOptionsBuilder::default();
-    client_common_config
-        .identity("integ_tester".to_owned())
-        .client_name("temporal-core".to_owned())
-        .client_version("0.1.0".to_owned());
-    let client1 = client_common_config
-        .clone()
-        .target_url(Url::parse(&format!("http://{}", server1.target)).unwrap())
-        .build()
-        .unwrap()
-        .connect("default", None)
+    let mut server1 = server_config
+        .start_server_with_output(Stdio::null(), Stdio::null())
         .await
         .unwrap();
-    let client2 = client_common_config
-        .clone()
-        .target_url(Url::parse(&format!("http://{}", server2.target)).unwrap())
-        .build()
-        .unwrap()
-        .connect("default", None)
+    let mut server2 = server_config
+        .start_server_with_output(Stdio::null(), Stdio::null())
         .await
         .unwrap();
 
-    // Start a workflow on both servers
-    info!("Starting workflows");
-    let wf1 = client1
-        .start_workflow(
-            vec![],
-            "my-task-queue".to_owned(),
-            "my-workflow-1".to_owned(),
-            "my-workflow-type".to_owned(),
-            None,
-            WorkflowOptions::default(),
-        )
-        .await
-        .unwrap();
-    let wf2 = client2
-        .start_workflow(
-            vec![],
-            "my-task-queue".to_owned(),
-            "my-workflow-2".to_owned(),
-            "my-workflow-type".to_owned(),
-            None,
-            WorkflowOptions::default(),
+    let result = std::panic::AssertUnwindSafe(async {
+        // Connect clients to both servers
+        info!("Connecting clients");
+        let mut client_common_config = ClientOptionsBuilder::default();
+        client_common_config
+            .identity("integ_tester".to_owned())
+            .client_name("temporal-core".to_owned())
+            .client_version("0.1.0".to_owned());
+        let client1 = client_common_config
+            .clone()
+            .target_url(Url::parse(&format!("http://{}", server1.target)).unwrap())
+            .build()
+            .unwrap()
+            .connect("default", None)
+            .await
+            .unwrap();
+        let client2 = client_common_config
+            .clone()
+            .target_url(Url::parse(&format!("http://{}", server2.target)).unwrap())
+            .build()
+            .unwrap()
+            .connect("default", None)
+            .await
+            .unwrap();
+
+        // Start a workflow on both servers
+        info!("Starting workflows");
+        let wf1 = client1
+            .start_workflow(
+                vec![],
+                "my-task-queue".to_owned(),
+                "my-workflow-1".to_owned(),
+                "my-workflow-type".to_owned(),
+                None,
+                WorkflowOptions::default(),
+            )
+            .await
+            .unwrap();
+        let wf2 = client2
+            .start_workflow(
+                vec![],
+                "my-task-queue".to_owned(),
+                "my-workflow-2".to_owned(),
+                "my-workflow-type".to_owned(),
+                None,
+                WorkflowOptions::default(),
+            )
+            .await
+            .unwrap();
+
+        // Create a worker only on the first server
+        let worker = init_worker(
+            init_integ_telem().unwrap(),
+            integ_worker_config("my-task-queue")
+                // We want a cache so we don't get extra remove-job activations
+                .max_cached_workflows(100_usize)
+                .build()
+                .unwrap(),
+            client1.clone(),
         )
-        .await
         .unwrap();
 
-    // Create a worker only on the first server
-    let worker = init_worker(
-        init_integ_telem().unwrap(),
-        integ_worker_config("my-task-queue")
-            // We want a cache so we don't get extra remove-job activations
-            .max_cached_workflows(100_usize)
-            .build()
-            .unwrap(),
-        client1.clone(),
-    )
-    .unwrap();
+        // Poll for first task, confirm it's first wf, complete, and wait for complete
+        info!("Doing initial poll");
+        let act1 = worker.poll_workflow_activation().await.unwrap();
+        assert_eq!(wf1.run_id, act1.run_id);
+        worker.complete_execution(&act1.run_id).await;
+        worker.handle_eviction().await;
+        info!("Waiting on first workflow complete");
+        client1
+            .get_untyped_workflow_handle("my-workflow-1", wf1.run_id)
+            .get_workflow_result(Default::default())
+            .await
+            .unwrap();
 
-    // Poll for first task, confirm it's first wf, complete, and wait for complete
-    info!("Doing initial poll");
-    let act1 = worker.poll_workflow_activation().await.unwrap();
-    assert_eq!(wf1.run_id, act1.run_id);
-    worker.complete_execution(&act1.run_id).await;
-    worker.handle_eviction().await;
-    info!("Waiting on first workflow complete");
-    client1
-        .get_untyped_workflow_handle("my-workflow-1", wf1.run_id)
-        .get_workflow_result(Default::default())
-        .await
-        .unwrap();
+        // Swap client, poll for next task, confirm it's second wf, and respond w/ empty
+        info!("Replacing client and polling again");
+        worker.replace_client(client2.get_client().inner().clone());
+        let act2 = worker.poll_workflow_activation().await.unwrap();
+        assert_eq!(wf2.run_id, act2.run_id);
+        worker.complete_execution(&act2.run_id).await;
+        worker.handle_eviction().await;
+        info!("Waiting on second workflow complete");
+        client2
+            .get_untyped_workflow_handle("my-workflow-2", wf2.run_id)
+            .get_workflow_result(Default::default())
+            .await
+            .unwrap();
 
-    // Swap client, poll for next task, confirm it's second wf, and respond w/ empty
-    info!("Replacing client and polling again");
-    worker.replace_client(client2.get_client().inner().clone());
-    let act2 = worker.poll_workflow_activation().await.unwrap();
-    assert_eq!(wf2.run_id, act2.run_id);
-    worker.complete_execution(&act2.run_id).await;
-    worker.handle_eviction().await;
-    info!("Waiting on second workflow complete");
-    client2
-        .get_untyped_workflow_handle("my-workflow-2", wf2.run_id)
-        .get_workflow_result(Default::default())
-        .await
-        .unwrap();
+        // Shutdown workers and servers
+        drain_pollers_and_shutdown(&(Arc::new(worker) as Arc<dyn Worker>)).await;
+    })
+    .catch_unwind()
+    .await;
 
-    // Shutdown workers and servers
-    drain_pollers_and_shutdown(&(Arc::new(worker) as Arc<dyn Worker>)).await;
-    server1.shutdown().await.unwrap();
-    server2.shutdown().await.unwrap();
+    let shutdown_results = join_all([server1.shutdown(), server2.shutdown()]).await;
+    if let Err(e) = result {
+        std::panic::resume_unwind(e);
+    }
+    for r in shutdown_results {
+        r.unwrap();
+    }
 }
 
 #[rstest::rstest]
@@ -292,3 +328,185 @@ async fn small_workflow_slots_and_pollers(#[values(false, true)] use_autoscaling
         .any(|e| e.event_type() == EventType::WorkflowTaskTimedOut);
     assert!(!any_task_timeouts);
 }
+
+#[tokio::test]
+async fn replace_client_works_after_polling_failure() {
+    let (log_consumer, mut log_rx) = CoreLogStreamConsumer::new(100);
+    let telem_opts = TelemetryOptionsBuilder::default()
+        .logging(Logger::Push {
+            filter: "OFF,temporal_client=DEBUG".into(),
+            consumer: Arc::new(log_consumer),
+        })
+        .build()
+        .unwrap();
+    let rt = Arc::new(CoreRuntime::new_assume_tokio(telem_opts).unwrap());
+
+    // Spawning background task to read logs and notify the test when polling failure occurs.
+    let look_for_poll_failure_log = Arc::new(AtomicBool::new(false));
+    let poll_retry_log_found = Arc::new(Notify::new());
+    let log_reader_join_handle = tokio::spawn({
+        let look_for_poll_retry_log = look_for_poll_failure_log.clone();
+        let poll_retry_log_found = poll_retry_log_found.clone();
+        async move {
+            let mut enabled = false;
+            loop {
+                let Some(log) = log_rx.next().await else {
+                    break;
+                };
+                if !enabled {
+                    enabled = look_for_poll_retry_log.load(Ordering::Acquire);
+                }
+                if enabled
+                    && (log
+                        .message
+                        .starts_with("gRPC call poll_workflow_task_queue failed")
+                        || log
+                            .message
+                            .starts_with("gRPC call poll_workflow_task_queue retried"))
+                {
+                    poll_retry_log_found.notify_one();
+                    break;
+                }
+            }
+        }
+    });
+    let abort_handles = Arc::new(Mutex::new(vec![log_reader_join_handle.abort_handle()]));
+
+    // Starting a second dev server for the worker to connect to initially. Later this server will be shut down
+    // and the worker client replaced with a client connected to the main integration test server.
+    let initial_server_config = integ_dev_server_config(vec![]).build().unwrap();
+    let initial_server = Arc::new(Mutex::new(Some(
+        initial_server_config
+            .start_server_with_output(Stdio::null(), Stdio::null())
+            .await
+            .unwrap(),
+    )));
+
+    let result = {
+        let initial_server = initial_server.clone();
+        let abort_handles = abort_handles.clone();
+        std::panic::AssertUnwindSafe(async move {
+            let initial_server_target = format!(
+                "http://{}",
+                initial_server.lock().unwrap().as_ref().unwrap().target
+            );
+            let client_for_initial_server = ClientOptionsBuilder::default()
+                .identity("client_for_initial_server".to_string())
+                .target_url(Url::parse(&initial_server_target).unwrap())
+                .client_name(INTEG_CLIENT_NAME.to_string())
+                .client_version(INTEG_CLIENT_VERSION.to_string())
+                .build()
+                .unwrap()
+                .connect(NAMESPACE, rt.telemetry().get_temporal_metric_meter())
+                .await
+                .unwrap();
+
+            let wf_name = "replace_client_works_after_polling_failure";
+            let task_queue = format!("{wf_name}_tq");
+
+            let worker = Arc::new(
+                init_worker(
+                    &rt,
+                    integ_worker_config(&task_queue)
+                        .max_cached_workflows(100_usize)
+                        .build()
+                        .unwrap(),
+                    client_for_initial_server.clone(),
+                )
+                .unwrap(),
+            );
+
+            // Polling the initial server the first time is successful.
+            let wf_1 = client_for_initial_server
+                .start_workflow(
+                    vec![],
+                    task_queue.clone(),
+                    wf_name.into(),
+                    wf_name.into(),
+                    None,
+                    WorkflowOptions::default(),
+                )
+                .await
+                .unwrap();
+            let act_1 =
+                tokio::time::timeout(Duration::from_secs(60), worker.poll_workflow_activation())
+                    .await
+                    .unwrap()
+                    .unwrap();
+            assert_eq!(act_1.run_id, wf_1.run_id);
+
+            // Initial server is shut down.
+            let mut server = initial_server.lock().unwrap().take().unwrap();
+            server.shutdown().await.unwrap();
+
+            // Start polling in a background task.
+            look_for_poll_failure_log.store(true, Ordering::Release);
+            let poll_join_handle = tokio::spawn({
+                let worker = worker.clone();
+                async move { worker.poll_workflow_activation().await }
+            });
+            abort_handles
+                .try_lock()
+                .unwrap()
+                .push(poll_join_handle.abort_handle());
+
+            // Wait until polling failure is detected.
+            tokio::time::timeout(Duration::from_secs(60), poll_retry_log_found.notified())
+                .await
+                .unwrap();
+
+            // Start a new WF on main integration server.
+            let client_for_integ_server = get_integ_server_options()
+                .connect(NAMESPACE, rt.telemetry().get_temporal_metric_meter())
+                .await
+                .unwrap();
+            let wf_2 = client_for_integ_server
+                .start_workflow(
+                    vec![],
+                    task_queue,
+                    wf_name.into(),
+                    wf_name.into(),
+                    None,
+                    WorkflowOptions {
+                        execution_timeout: Some(Duration::from_secs(60)),
+                        ..Default::default()
+                    },
+                )
+                .await
+                .unwrap();
+
+            // Switch worker over to the main integration server.
+            // The polling started on the initial server should complete with a task from the new server.
+            worker.replace_client(client_for_integ_server);
+            let act_2 = tokio::time::timeout(Duration::from_secs(60), poll_join_handle)
+                .await
+                .unwrap()
+                .unwrap()
+                .unwrap();
+            assert_eq!(act_2.run_id, wf_2.run_id);
+        })
+    }
+    .catch_unwind()
+    .await;
+
+    // Cleaning up spawned background tasks if they're still running.
+    for handle in &*abort_handles.lock().unwrap() {
+        handle.abort();
+    }
+
+    // If the test panicked, we may or may not need to shut down the server here.
+    // If the test succeeded, the server should always be shut down by this point.
+    let server = initial_server.lock().unwrap().take();
+    if let Some(mut server) = server {
+        let _ = server.shutdown().await;
+        assert_matches!(
+            result,
+            Err(_),
+            "Server should have been shut down during the test"
+        );
+    }
+
+    if let Err(e) = result {
+        std::panic::resume_unwind(e);
+    }
+}

package/sdk-core/tests/integ_tests/queries_tests.rs

@@ -1,18 +1,18 @@
+use crate::common::{CoreWfStarter, init_core_and_create_wf};
 use assert_matches::assert_matches;
 use futures_util::{FutureExt, StreamExt, future::join_all, stream::FuturesUnordered};
 use std::time::{Duration, Instant};
 use temporal_client::WorkflowClientTrait;
+use temporal_sdk_core::test_help::{WorkerTestHelpers, drain_pollers_and_shutdown};
 use temporal_sdk_core_protos::{
     coresdk::{
         workflow_activation::{WorkflowActivationJob, workflow_activation_job},
         workflow_commands::{QueryResult, QuerySuccess, StartTimer},
         workflow_completion::WorkflowActivationCompletion,
     },
+    prost_dur,
     temporal::api::{failure::v1::Failure, query::v1::WorkflowQuery},
-};
-use temporal_sdk_core_test_utils::{
-    CoreWfStarter, WorkerTestHelpers, drain_pollers_and_shutdown, init_core_and_create_wf,
-    start_timer_cmd,
+    test_utils::start_timer_cmd,
 };
 use tokio::join;
 

package/sdk-core/tests/integ_tests/update_tests.rs

@@ -1,3 +1,6 @@
+use crate::common::{
+    CoreWfStarter, WorkflowHandleExt, init_core_and_create_wf, init_core_replay_preloaded,
+};
 use anyhow::anyhow;
 use assert_matches::assert_matches;
 use futures_util::{StreamExt, future, future::join_all};
@@ -12,7 +15,10 @@ use temporal_client::{
     Client, NamespacedClient, RetryClient, WorkflowClientTrait, WorkflowService,
 };
 use temporal_sdk::{ActContext, ActivityOptions, LocalActivityOptions, UpdateContext, WfContext};
-use temporal_sdk_core::replay::HistoryForReplay;
+use temporal_sdk_core::{
+    replay::HistoryForReplay,
+    test_help::{WorkerTestHelpers, drain_pollers_and_shutdown},
+};
 use temporal_sdk_core_api::Worker;
 use temporal_sdk_core_protos::{
     coresdk::{
@@ -26,18 +32,17 @@ use temporal_sdk_core_protos::{
         },
         workflow_completion::WorkflowActivationCompletion,
     },
+    prost_dur,
     temporal::api::{
         common::v1::WorkflowExecution,
         enums::v1::{EventType, ResetReapplyType, UpdateWorkflowExecutionLifecycleStage},
         update::{self, v1::WaitPolicy},
         workflowservice::v1::ResetWorkflowExecutionRequest,
     },
-};
-use temporal_sdk_core_test_utils::{
-    CoreWfStarter, WorkerTestHelpers, WorkflowHandleExt, drain_pollers_and_shutdown,
-    init_core_and_create_wf, init_core_replay_preloaded, start_timer_cmd,
+    test_utils::start_timer_cmd,
 };
 use tokio::{join, sync::Barrier};
+use tonic::IntoRequest;
 use uuid::Uuid;
 
 #[derive(Clone, Copy)]
@@ -112,7 +117,7 @@ async fn reapplied_updates_due_to_reset() {
         Arc::make_mut(&mut client_mut),
         #[allow(deprecated)]
         ResetWorkflowExecutionRequest {
-            namespace: client.namespace().into(),
+            namespace: client.namespace(),
             workflow_execution: Some(WorkflowExecution {
                 workflow_id: workflow_id.into(),
                 run_id: pre_reset_run_id.clone(),
@@ -121,7 +126,8 @@ async fn reapplied_updates_due_to_reset() {
             reset_reapply_type: ResetReapplyType::AllEligible as i32,
             request_id: Uuid::new_v4().to_string(),
             ..Default::default()
-        },
+        }
+        .into_request(),
     )
     .await
     .unwrap()

package/sdk-core/tests/integ_tests/visibility_tests.rs

@@ -1,16 +1,14 @@
+use crate::common::{CoreWfStarter, NAMESPACE, eventually, get_integ_server_options};
 use assert_matches::assert_matches;
 use std::{sync::Arc, time::Duration};
 use temporal_client::{
     ListClosedFilters, ListOpenFilters, Namespace, RegisterNamespaceOptions, StartTimeFilter,
     WorkflowClientTrait, WorkflowExecutionFilter,
 };
+use temporal_sdk_core::test_help::{WorkerTestHelpers, drain_pollers_and_shutdown};
 use temporal_sdk_core_protos::coresdk::workflow_activation::{
     WorkflowActivationJob, workflow_activation_job,
 };
-use temporal_sdk_core_test_utils::{
-    CoreWfStarter, NAMESPACE, WorkerTestHelpers, drain_pollers_and_shutdown,
-    get_integ_server_options,
-};
 use tokio::time::sleep;
 
 #[tokio::test]
@@ -42,11 +40,30 @@ async fn client_list_open_closed_workflow_executions() {
         workflow_id: wf_name.clone(),
         run_id: "".to_owned(),
     });
-    let open_workflows = client
-        .list_open_workflow_executions(1, Default::default(), Some(start_time_filter), Some(filter))
-        .await
-        .unwrap();
-    assert_eq!(open_workflows.executions.len(), 1);
+    let open_workflows = eventually(
+        || async {
+            let open_workflows = client
+                .list_open_workflow_executions(
+                    1,
+                    Default::default(),
+                    Some(start_time_filter),
+                    Some(filter.clone()),
+                )
+                .await
+                .unwrap();
+            if open_workflows.executions.len() == 1 {
+                Ok(open_workflows)
+            } else {
+                Err(format!(
+                    "Expected 1 open workflow, got {}",
+                    open_workflows.executions.len()
+                ))
+            }
+        },
+        Duration::from_secs(5),
+    )
+    .await
+    .unwrap();
     let workflow = open_workflows.executions[0].clone();
     assert_eq!(workflow.execution.as_ref().unwrap().workflow_id, wf_name);