@temporalio/core-bridge 1.9.2 → 1.10.0

package/sdk-core/tests/heavy_tests.rs

@@ -1,9 +1,14 @@
 use futures::{future::join_all, sink, stream::FuturesUnordered, StreamExt};
-use std::time::{Duration, Instant};
+use std::{
+    sync::Arc,
+    time::{Duration, Instant},
+};
 use temporal_client::{WfClientExt, WorkflowClientTrait, WorkflowOptions};
 use temporal_sdk::{ActContext, ActivityOptions, WfContext, WorkflowResult};
-use temporal_sdk_core_protos::coresdk::{
-    workflow_commands::ActivityCancellationType, AsJsonPayloadExt,
+use temporal_sdk_core::{ResourceBasedSlots, ResourceBasedTuner, ResourceSlotOptions};
+use temporal_sdk_core_protos::{
+    coresdk::{workflow_commands::ActivityCancellationType, AsJsonPayloadExt},
+    temporal::api::enums::v1::WorkflowIdReusePolicy,
 };
 use temporal_sdk_core_test_utils::{workflows::la_problem_workflow, CoreWfStarter};
 
@@ -15,10 +20,11 @@ async fn activity_load() {
 
     let mut starter = CoreWfStarter::new("activity_load");
     starter
-        .max_wft(CONCURRENCY)
+        .worker_config
+        .max_outstanding_workflow_tasks(CONCURRENCY)
         .max_cached_workflows(CONCURRENCY)
-        .max_at_polls(10)
-        .max_at(CONCURRENCY);
+        .max_concurrent_at_polls(10_usize)
+        .max_outstanding_activities(CONCURRENCY);
     let mut worker = starter.worker().await;
 
     let activity_id = "act-1";
@@ -78,24 +84,111 @@ async fn activity_load() {
     dbg!(running.elapsed());
 }
 
+#[tokio::test]
+async fn chunky_activities_resource_based() {
+    const WORKFLOWS: usize = 100;
+
+    let mut starter = CoreWfStarter::new("chunky_activities_resource_based");
+    starter
+        .worker_config
+        .clear_max_outstanding_opts()
+        .max_concurrent_wft_polls(10_usize)
+        .max_concurrent_at_polls(10_usize);
+    let mut tuner = ResourceBasedTuner::new(ResourceBasedSlots::new(0.7, 0.7));
+    tuner
+        .with_workflow_slots_options(ResourceSlotOptions::new(
+            25,
+            WORKFLOWS,
+            Duration::from_millis(0),
+        ))
+        .with_activity_slots_options(ResourceSlotOptions::new(5, 1000, Duration::from_millis(50)));
+    starter.worker_config.tuner(Arc::new(tuner));
+    let mut worker = starter.worker().await;
+
+    let activity_id = "act-1";
+    let activity_timeout = Duration::from_secs(30);
+
+    let wf_fn = move |ctx: WfContext| {
+        let payload = "yo".as_json_payload().unwrap();
+        async move {
+            let activity = ActivityOptions {
+                activity_id: Some(activity_id.to_string()),
+                activity_type: "test_activity".to_string(),
+                input: payload.clone(),
+                start_to_close_timeout: Some(activity_timeout),
+                ..Default::default()
+            };
+            let res = ctx.activity(activity).await.unwrap_ok_payload();
+            assert_eq!(res.data, payload.data);
+            Ok(().into())
+        }
+    };
+
+    let starting = Instant::now();
+    let wf_type = "chunky_activity_wf";
+    worker.register_wf(wf_type.to_owned(), wf_fn);
+    worker.register_activity(
+        "test_activity",
+        |_ctx: ActContext, echo: String| async move {
+            tokio::task::spawn_blocking(move || {
+                // Allocate a gig and then do some CPU stuff on it
+                let mut mem = vec![0_u8; 1000 * 1024 * 1024];
+                for _ in 1..10 {
+                    for i in 0..mem.len() {
+                        mem[i] &= mem[mem.len() - 1 - i]
+                    }
+                }
+                Ok(echo)
+            })
+            .await?
+        },
+    );
+    join_all((0..WORKFLOWS).map(|i| {
+        let worker = &worker;
+        let wf_id = format!("chunk_activity_{i}");
+        async move {
+            worker
+                .submit_wf(
+                    wf_id,
+                    wf_type.to_owned(),
+                    vec![],
+                    WorkflowOptions {
+                        id_reuse_policy: WorkflowIdReusePolicy::TerminateIfRunning,
+                        ..Default::default()
+                    },
+                )
+                .await
+                .unwrap();
+        }
+    }))
+    .await;
+    dbg!(starting.elapsed());
+
+    let running = Instant::now();
+
+    worker.run_until_done().await.unwrap();
+    dbg!(running.elapsed());
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
 async fn workflow_load() {
     const SIGNAME: &str = "signame";
-    let num_workflows = 200;
+    let num_workflows = 500;
     let wf_name = "workflow_load";
     let mut starter = CoreWfStarter::new("workflow_load");
     starter
-        .max_wft(5)
-        .max_cached_workflows(5)
-        .max_at_polls(10)
-        .max_at(100);
+        .worker_config
+        .max_outstanding_workflow_tasks(5_usize)
+        .max_cached_workflows(200_usize)
+        .max_concurrent_at_polls(10_usize)
+        .max_outstanding_activities(100_usize);
     let mut worker = starter.worker().await;
     worker.register_wf(wf_name.to_owned(), |ctx: WfContext| async move {
         let sigchan = ctx.make_signal_channel(SIGNAME).map(Ok);
         let drained_fut = sigchan.forward(sink::drain());
 
         let real_stuff = async move {
-            for _ in 0..20 {
+            for _ in 0..5 {
                 ctx.activity(ActivityOptions {
                     activity_type: "echo_activity".to_string(),
                     start_to_close_timeout: Some(Duration::from_secs(5)),
@@ -162,8 +255,10 @@ async fn workflow_load() {
 async fn evict_while_la_running_no_interference() {
     let wf_name = "evict_while_la_running_no_interference";
     let mut starter = CoreWfStarter::new(wf_name);
-    starter.max_local_at(20);
-    starter.max_cached_workflows(20);
+    starter
+        .worker_config
+        .max_outstanding_local_activities(20_usize)
+        .max_cached_workflows(20_usize);
     // Though it doesn't make sense to set wft higher than cached workflows, leaving this commented
     // introduces more instability that can be useful in the test.
     // starter.max_wft(20);
@@ -228,9 +323,11 @@ pub async fn many_parallel_timers_longhist(ctx: WfContext) -> WorkflowResult<()>
 async fn can_paginate_long_history() {
     let wf_name = "can_paginate_long_history";
     let mut starter = CoreWfStarter::new(wf_name);
-    starter.no_remote_activities();
-    // Do not use sticky queues so we are forced to paginate once history gets long
-    starter.max_cached_workflows(0);
+    starter
+        .worker_config
+        .no_remote_activities(true)
+        // Do not use sticky queues so we are forced to paginate once history gets long
+        .max_cached_workflows(0_usize);
 
     let mut worker = starter.worker().await;
     worker.register_wf(wf_name.to_owned(), many_parallel_timers_longhist);

package/sdk-core/tests/integ_tests/activity_functions.rs

@@ -1,5 +1,5 @@
 use temporal_sdk::ActContext;
 
-pub async fn echo(_ctx: ActContext, e: String) -> anyhow::Result<String> {
+pub(crate) async fn echo(_ctx: ActContext, e: String) -> anyhow::Result<String> {
     Ok(e)
 }

package/sdk-core/tests/integ_tests/client_tests.rs

@@ -17,7 +17,7 @@ async fn can_use_retry_client() {
 #[tokio::test]
 async fn can_use_retry_raw_client() {
     let opts = get_integ_server_options();
-    let raw_client = opts.connect_no_namespace(None, None).await.unwrap();
+    let raw_client = opts.connect_no_namespace(None).await.unwrap();
     let mut retry_client = RetryClient::new(raw_client, opts.retry_config);
     retry_client
         .describe_namespace(DescribeNamespaceRequest {
@@ -31,6 +31,6 @@ async fn can_use_retry_raw_client() {
 #[tokio::test]
 async fn calls_get_system_info() {
     let opts = get_integ_server_options();
-    let raw_client = opts.connect_no_namespace(None, None).await.unwrap();
+    let raw_client = opts.connect_no_namespace(None).await.unwrap();
     assert!(raw_client.get_client().capabilities().is_some());
 }

package/sdk-core/tests/integ_tests/ephemeral_server_tests.rs

@@ -1,8 +1,10 @@
+use futures::stream;
+use futures::TryStreamExt;
 use std::time::{SystemTime, UNIX_EPOCH};
 use temporal_client::{ClientOptionsBuilder, TestService, WorkflowService};
 use temporal_sdk_core::ephemeral_server::{
     EphemeralExe, EphemeralExeVersion, EphemeralServer, TemporalDevServerConfigBuilder,
-    TemporaliteConfigBuilder, TestServerConfigBuilder,
+    TestServerConfigBuilder,
 };
 use temporal_sdk_core_protos::temporal::api::workflowservice::v1::DescribeNamespaceRequest;
 use temporal_sdk_core_test_utils::{default_cached_download, NAMESPACE};
@@ -16,7 +18,12 @@ async fn temporal_cli_default() {
         .unwrap();
     let mut server = config.start_server().await.unwrap();
     assert_ephemeral_server(&server).await;
+
+    // Make sure process is there on start and not there after shutdown
+    let pid = sysinfo::Pid::from_u32(server.child_process_id().unwrap());
+    assert!(sysinfo::System::new_all().process(pid).is_some());
     server.shutdown().await.unwrap();
+    assert!(sysinfo::System::new_all().process(pid).is_none());
 }
 
 #[tokio::test]
@@ -31,32 +38,10 @@ async fn temporal_cli_fixed() {
 }
 
 #[tokio::test]
-async fn temporalite_default() {
-    let config = TemporaliteConfigBuilder::default()
-        .exe(default_cached_download())
-        .build()
-        .unwrap();
-    let mut server = config.start_server().await.unwrap();
-    assert_ephemeral_server(&server).await;
-    server.shutdown().await.unwrap();
-}
-
-#[tokio::test]
-async fn temporalite_fixed() {
-    let config = TemporaliteConfigBuilder::default()
-        .exe(fixed_cached_download("v0.2.0"))
-        .build()
-        .unwrap();
-    let mut server = config.start_server().await.unwrap();
-    assert_ephemeral_server(&server).await;
-    server.shutdown().await.unwrap();
-}
-
-#[tokio::test]
-async fn temporalite_shutdown_port_reuse() {
+async fn temporal_cli_shutdown_port_reuse() {
     // Start, test shutdown, do again immediately on same port to ensure we can
     // reuse after shutdown
-    let config = TemporaliteConfigBuilder::default()
+    let config = TemporalDevServerConfigBuilder::default()
         .exe(default_cached_download())
         .port(Some(10123))
         .build()
@@ -69,6 +54,33 @@ async fn temporalite_shutdown_port_reuse() {
     server.shutdown().await.unwrap();
 }
 
+// This test will fail on Linux until https://github.com/temporalio/cli/pull/564
+// gets released (presumably in 0.12.1). To test locally, build CLI manually
+// and use that specific binary instead:
+// ```
+//   .exe(EphemeralExe::ExistingPath(
+//       "/usr/local/bin/temporal".to_string(),
+//   ))
+// ```
+#[tokio::test]
+#[ignore]
+async fn temporal_cli_concurrent_starts() -> Result<(), Box<dyn std::error::Error>> {
+    stream::iter((0..80).map(|_| {
+        TemporalDevServerConfigBuilder::default()
+            .exe(default_cached_download())
+            .build()
+            .map_err(anyhow::Error::from)
+    }))
+    .try_for_each_concurrent(8, |config| async move {
+        let mut server = config.start_server().await?;
+        server.shutdown().await?;
+        Ok(())
+    })
+    .await?;
+
+    Ok(())
+}
+
 #[tokio::test]
 async fn test_server_default() {
     let config = TestServerConfigBuilder::default()
@@ -124,7 +136,7 @@ async fn assert_ephemeral_server(server: &EphemeralServer) {
         .client_version("0.1.0".to_string())
         .build()
         .unwrap()
-        .connect_no_namespace(None, None)
+        .connect_no_namespace(None)
         .await
         .unwrap();
     let resp = client

package/sdk-core/tests/integ_tests/metrics_tests.rs

@@ -23,8 +23,10 @@ use temporal_sdk_core_protos::{
         ActivityTaskCompletion,
     },
     temporal::api::{
-        enums::v1::WorkflowIdReusePolicy, failure::v1::Failure, query::v1::WorkflowQuery,
-        workflowservice::v1::ListNamespacesRequest,
+        enums::v1::WorkflowIdReusePolicy,
+        failure::v1::Failure,
+        query::v1::WorkflowQuery,
+        workflowservice::v1::{DescribeNamespaceRequest, ListNamespacesRequest},
     },
 };
 use temporal_sdk_core_test_utils::{
@@ -34,24 +36,30 @@ use tokio::{join, sync::Barrier, task::AbortHandle};
 
 static ANY_PORT: &str = "127.0.0.1:0";
 
-async fn get_text(endpoint: String) -> String {
+pub(crate) async fn get_text(endpoint: String) -> String {
     reqwest::get(endpoint).await.unwrap().text().await.unwrap()
 }
 
-struct AbortOnDrop {
+pub(crate) struct AbortOnDrop {
     ah: AbortHandle,
 }
+
 impl Drop for AbortOnDrop {
     fn drop(&mut self) {
         self.ah.abort();
     }
 }
 
-fn prom_metrics() -> (TelemetryOptions, SocketAddr, AbortOnDrop) {
+pub(crate) fn prom_metrics(
+    use_seconds: bool,
+    show_units: bool,
+) -> (TelemetryOptions, SocketAddr, AbortOnDrop) {
     let mut telemopts = get_integ_telem_options();
     let prom_info = start_prometheus_metric_exporter(
         PrometheusExporterOptionsBuilder::default()
             .socket_addr(ANY_PORT.parse().unwrap())
+            .use_seconds_for_durations(use_seconds)
+            .unit_suffix(show_units)
             .build()
             .unwrap(),
     )
@@ -66,13 +74,14 @@ fn prom_metrics() -> (TelemetryOptions, SocketAddr, AbortOnDrop) {
     )
 }
 
+#[rstest::rstest]
 #[tokio::test]
-async fn prometheus_metrics_exported() {
-    let (telemopts, addr, _aborter) = prom_metrics();
+async fn prometheus_metrics_exported(#[values(true, false)] use_seconds_latency: bool) {
+    let (telemopts, addr, _aborter) = prom_metrics(use_seconds_latency, false);
     let rt = CoreRuntime::new_assume_tokio(telemopts).unwrap();
     let opts = get_integ_server_options();
     let mut raw_client = opts
-        .connect_no_namespace(rt.telemetry().get_temporal_metric_meter(), None)
+        .connect_no_namespace(rt.telemetry().get_temporal_metric_meter())
         .await
         .unwrap();
     assert!(raw_client.get_client().capabilities().is_some());
@@ -89,6 +98,17 @@ async fn prometheus_metrics_exported() {
     assert!(body.contains(
         "temporal_request_latency_count{operation=\"GetSystemInfo\",service_name=\"temporal-core-sdk\"} 1"
     ));
+    if use_seconds_latency {
+        assert!(body.contains(
+            "temporal_request_latency_bucket{\
+             operation=\"GetSystemInfo\",service_name=\"temporal-core-sdk\",le=\"0.05\"}"
+        ));
+    } else {
+        assert!(body.contains(
+            "temporal_request_latency_bucket{\
+             operation=\"GetSystemInfo\",service_name=\"temporal-core-sdk\",le=\"50\"}"
+        ));
+    }
     // Verify counter names are appropriate (don't end w/ '_total')
     assert!(body.contains("temporal_request{"));
     // Verify non-temporal metrics meter does not prefix
@@ -106,7 +126,7 @@ async fn prometheus_metrics_exported() {
 
 #[tokio::test]
 async fn one_slot_worker_reports_available_slot() {
-    let (telemopts, addr, _aborter) = prom_metrics();
+    let (telemopts, addr, _aborter) = prom_metrics(false, false);
     let tq = "one_slot_worker_tq";
     let rt = CoreRuntime::new_assume_tokio(telemopts).unwrap();
 
@@ -125,7 +145,7 @@ async fn one_slot_worker_reports_available_slot() {
 
     let client = Arc::new(
         get_integ_server_options()
-            .connect(worker_cfg.namespace.clone(), None, None)
+            .connect(worker_cfg.namespace.clone(), None)
             .await
             .expect("Must connect"),
     );
@@ -235,8 +255,7 @@ async fn one_slot_worker_reports_available_slot() {
 
         wf_task_barr.wait().await;
 
-        // At this point the workflow task is outstanding, so there should be 0 slots, and
-        // the activities haven't started, so there should still be 1 each.
+        // At this point the workflow task is outstanding and the activities haven't started
         let body = get_text(format!("http://{addr}/metrics")).await;
         assert!(body.contains(&format!(
             "temporal_worker_task_slots_available{{namespace=\"{NAMESPACE}\",\
@@ -253,8 +272,23 @@ async fn one_slot_worker_reports_available_slot() {
              service_name=\"temporal-core-sdk\",task_queue=\"one_slot_worker_tq\",\
              worker_type=\"LocalActivityWorker\"}} 1"
         )));
+        assert!(body.contains(&format!(
+            "temporal_worker_task_slots_used{{namespace=\"{NAMESPACE}\",\
+             service_name=\"temporal-core-sdk\",task_queue=\"one_slot_worker_tq\",\
+             worker_type=\"WorkflowWorker\"}} 1"
+        )));
+        assert!(body.contains(&format!(
+            "temporal_worker_task_slots_used{{namespace=\"{NAMESPACE}\",\
+             service_name=\"temporal-core-sdk\",task_queue=\"one_slot_worker_tq\",\
+             worker_type=\"ActivityWorker\"}} 0"
+        )));
+        assert!(body.contains(&format!(
+            "temporal_worker_task_slots_used{{namespace=\"{NAMESPACE}\",\
+             service_name=\"temporal-core-sdk\",task_queue=\"one_slot_worker_tq\",\
+             worker_type=\"LocalActivityWorker\"}} 0"
+        )));
 
-        // Now we allow the complete to proceed. Once it goes through, there should be 1 WFT slot
+        // Now we allow the complete to proceed. Once it goes through, there should be 2 WFT slot
         // open but 0 activity slots
         wf_task_barr.wait().await;
         wf_task_barr.wait().await;
@@ -271,6 +305,11 @@ async fn one_slot_worker_reports_available_slot() {
              service_name=\"temporal-core-sdk\",task_queue=\"one_slot_worker_tq\",\
              worker_type=\"ActivityWorker\"}} 0"
         )));
+        assert!(body.contains(&format!(
+            "temporal_worker_task_slots_used{{namespace=\"{NAMESPACE}\",\
+             service_name=\"temporal-core-sdk\",task_queue=\"one_slot_worker_tq\",\
+             worker_type=\"ActivityWorker\"}} 1"
+        )));
 
         // Now complete the activity and watch it go up
         act_task_barr.wait().await;
@@ -292,6 +331,11 @@ async fn one_slot_worker_reports_available_slot() {
              service_name=\"temporal-core-sdk\",task_queue=\"one_slot_worker_tq\",\
              worker_type=\"LocalActivityWorker\"}} 0"
         )));
+        assert!(body.contains(&format!(
+            "temporal_worker_task_slots_used{{namespace=\"{NAMESPACE}\",\
+             service_name=\"temporal-core-sdk\",task_queue=\"one_slot_worker_tq\",\
+             worker_type=\"LocalActivityWorker\"}} 1"
+        )));
         // When completion is done, we have 1 again
         act_task_barr.wait().await;
         act_task_barr.wait().await;
@@ -318,12 +362,12 @@ async fn query_of_closed_workflow_doesnt_tick_terminal_metric(
     )]
     completion: workflow_command::Variant,
 ) {
-    let (telemopts, addr, _aborter) = prom_metrics();
+    let (telemopts, addr, _aborter) = prom_metrics(false, false);
     let rt = CoreRuntime::new_assume_tokio(telemopts).unwrap();
     let mut starter =
         CoreWfStarter::new_with_runtime("query_of_closed_workflow_doesnt_tick_terminal_metric", rt);
     // Disable cache to ensure replay happens completely
-    starter.max_cached_workflows(0);
+    starter.worker_config.max_cached_workflows(0_usize);
     let worker = starter.get_worker().await;
     let run_id = starter.start_wf().await;
     let task = worker.poll_workflow_activation().await.unwrap();
@@ -447,13 +491,13 @@ fn runtime_new() {
     .unwrap();
     let handle = rt.tokio_handle();
     let _rt = handle.enter();
-    let (telemopts, addr, _aborter) = prom_metrics();
+    let (telemopts, addr, _aborter) = prom_metrics(false, false);
     rt.telemetry_mut()
         .attach_late_init_metrics(telemopts.metrics.unwrap());
     let opts = get_integ_server_options();
     handle.block_on(async {
         let mut raw_client = opts
-            .connect_no_namespace(rt.telemetry().get_temporal_metric_meter(), None)
+            .connect_no_namespace(rt.telemetry().get_temporal_metric_meter())
             .await
             .unwrap();
         assert!(raw_client.get_client().capabilities().is_some());
@@ -465,3 +509,68 @@ fn runtime_new() {
         assert!(body.contains("temporal_request"));
     });
 }
+
+#[rstest::rstest]
+#[tokio::test]
+async fn latency_metrics(
+    #[values(true, false)] use_seconds_latency: bool,
+    #[values(true, false)] show_units: bool,
+) {
+    let (telemopts, addr, _aborter) = prom_metrics(use_seconds_latency, show_units);
+    let rt = CoreRuntime::new_assume_tokio(telemopts).unwrap();
+    let mut starter = CoreWfStarter::new_with_runtime("latency_metrics", rt);
+    let worker = starter.get_worker().await;
+    starter.start_wf().await;
+    // Immediately finish workflow
+    let task = worker.poll_workflow_activation().await.unwrap();
+    worker
+        .complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
+            task.run_id,
+            CompleteWorkflowExecution { result: None }.into(),
+        ))
+        .await
+        .unwrap();
+
+    let body = get_text(format!("http://{addr}/metrics")).await;
+    let matching_line = body
+        .lines()
+        .find(|l| l.starts_with("temporal_workflow_endtoend_latency"))
+        .unwrap();
+
+    if use_seconds_latency {
+        if show_units {
+            assert!(matching_line.contains("temporal_workflow_endtoend_latency_seconds"));
+        }
+        assert!(matching_line.contains("le=\"0.1\""));
+    } else {
+        if show_units {
+            assert!(matching_line.contains("temporal_workflow_endtoend_latency_milliseconds"));
+        }
+        assert!(matching_line.contains("le=\"100\""));
+    }
+}
+
+#[tokio::test]
+async fn request_fail_codes() {
+    let (telemopts, addr, _aborter) = prom_metrics(false, false);
+    let rt = CoreRuntime::new_assume_tokio(telemopts).unwrap();
+    let opts = get_integ_server_options();
+    let mut client = opts
+        .connect(NAMESPACE, rt.telemetry().get_temporal_metric_meter())
+        .await
+        .unwrap();
+
+    // Describe namespace w/ invalid argument (unset namespace field)
+    WorkflowService::describe_namespace(&mut client, DescribeNamespaceRequest::default())
+        .await
+        .unwrap_err();
+
+    let body = get_text(format!("http://{addr}/metrics")).await;
+    let matching_line = body
+        .lines()
+        .find(|l| l.starts_with("temporal_request_failure"))
+        .unwrap();
+    assert!(matching_line.contains("operation=\"DescribeNamespace\""));
+    assert!(matching_line.contains("status_code=\"INVALID_ARGUMENT\""));
+    assert!(matching_line.contains("} 1"));
+}