@temporalio/core-bridge 1.13.1 → 1.14.0

package/sdk-core/{tests → crates/sdk-core/tests}/integ_tests/metrics_tests.rs

@@ -1,4 +1,10 @@
-use crate::integ_tests::mk_nexus_endpoint;
+use crate::{
+    common::{
+        ANY_PORT, CoreWfStarter, NAMESPACE, OTEL_URL_ENV_VAR, PROMETHEUS_QUERY_API,
+        get_integ_runtime_options, get_integ_server_options, get_integ_telem_options, prom_metrics,
+    },
+    integ_tests::mk_nexus_endpoint,
+};
 use anyhow::anyhow;
 use assert_matches::assert_matches;
 use std::{
@@ -8,23 +14,42 @@ use std::{
     sync::{Arc, OnceLock},
     time::Duration,
 };
-use temporal_client::{
+use temporalio_client::{
     REQUEST_LATENCY_HISTOGRAM_NAME, WorkflowClientTrait, WorkflowOptions, WorkflowService,
 };
-use temporal_sdk::{
-    ActContext, ActivityError, ActivityOptions, CancellableFuture, LocalActivityOptions,
-    NexusOperationOptions, WfContext,
-};
-use temporal_sdk_core::{
-    CoreRuntime, FixedSizeSlotSupplier, TokioRuntimeBuilder, TunerBuilder, init_worker,
-    telemetry::{WORKFLOW_TASK_EXECUTION_LATENCY_HISTOGRAM_NAME, build_otlp_metric_exporter},
-};
-use temporal_sdk_core_api::{
+use temporalio_common::{
     Worker,
     errors::PollError,
+    prost_dur,
+    protos::{
+        coresdk::{
+            ActivityTaskCompletion, AsJsonPayloadExt,
+            activity_result::ActivityExecutionResult,
+            nexus::{NexusTaskCompletion, nexus_task, nexus_task_completion},
+            workflow_activation::{WorkflowActivationJob, workflow_activation_job},
+            workflow_commands::{
+                CancelWorkflowExecution, CompleteWorkflowExecution, ContinueAsNewWorkflowExecution,
+                FailWorkflowExecution, QueryResult, QuerySuccess, ScheduleActivity,
+                ScheduleLocalActivity, workflow_command,
+            },
+            workflow_completion::WorkflowActivationCompletion,
+        },
+        temporal::api::{
+            common::v1::RetryPolicy,
+            enums::v1::{NexusHandlerErrorRetryBehavior, WorkflowIdReusePolicy},
+            failure::v1::Failure,
+            nexus,
+            nexus::v1::{
+                HandlerError, StartOperationResponse, UnsuccessfulOperationError, request::Variant,
+                start_operation_response,
+            },
+            query::v1::WorkflowQuery,
+            workflowservice::v1::{DescribeNamespaceRequest, ListNamespacesRequest},
+        },
+    },
     telemetry::{
         HistogramBucketOverrides, OtelCollectorOptionsBuilder, OtlpProtocol,
-        PrometheusExporterOptionsBuilder, TelemetryOptionsBuilder,
+        PrometheusExporterOptionsBuilder, TaskQueueLabelStrategy, TelemetryOptionsBuilder,
         metrics::{
             CoreMeter, CounterBase, Gauge, GaugeBase, HistogramBase, MetricKeyValue,
             MetricParameters, MetricParametersBuilder, NewAttributes,
@@ -32,41 +57,20 @@ use temporal_sdk_core_api::{
     },
     worker::{
         PollerBehavior, SlotKind, SlotMarkUsedContext, SlotReleaseContext, SlotReservationContext,
-        SlotSupplier, SlotSupplierPermit, WorkerConfigBuilder, WorkerVersioningStrategy,
-        WorkflowSlotKind,
+        SlotSupplier, SlotSupplierPermit, WorkerConfigBuilder, WorkerTaskTypes,
+        WorkerVersioningStrategy, WorkflowSlotKind,
     },
 };
-use temporal_sdk_core_protos::{
-    coresdk::{
-        ActivityTaskCompletion, AsJsonPayloadExt,
-        activity_result::ActivityExecutionResult,
-        nexus::{NexusTaskCompletion, nexus_task, nexus_task_completion},
-        workflow_activation::{WorkflowActivationJob, workflow_activation_job},
-        workflow_commands::{
-            CancelWorkflowExecution, CompleteWorkflowExecution, ContinueAsNewWorkflowExecution,
-            FailWorkflowExecution, QueryResult, QuerySuccess, ScheduleActivity,
-            ScheduleLocalActivity, workflow_command,
-        },
-        workflow_completion::WorkflowActivationCompletion,
-    },
-    temporal::api::{
-        common::v1::RetryPolicy,
-        enums::v1::{NexusHandlerErrorRetryBehavior, WorkflowIdReusePolicy},
-        failure::v1::Failure,
-        nexus,
-        nexus::v1::{
-            HandlerError, StartOperationResponse, UnsuccessfulOperationError, request::Variant,
-            start_operation_response,
-        },
-        query::v1::WorkflowQuery,
-        workflowservice::v1::{DescribeNamespaceRequest, ListNamespacesRequest},
-    },
+use temporalio_sdk::{
+    ActContext, ActivityError, ActivityOptions, CancellableFuture, LocalActivityOptions,
+    NexusOperationOptions, WfContext,
 };
-use temporal_sdk_core_test_utils::{
-    ANY_PORT, CoreWfStarter, NAMESPACE, OTEL_URL_ENV_VAR, PROMETHEUS_QUERY_API,
-    get_integ_server_options, get_integ_telem_options, prom_metrics,
+use temporalio_sdk_core::{
+    CoreRuntime, FixedSizeSlotSupplier, TokioRuntimeBuilder, TunerBuilder, init_worker,
+    telemetry::{WORKFLOW_TASK_EXECUTION_LATENCY_HISTOGRAM_NAME, build_otlp_metric_exporter},
 };
 use tokio::{join, sync::Barrier};
+use tonic::IntoRequest;
 use url::Url;
 
 pub(crate) async fn get_text(endpoint: String) -> String {
@@ -94,7 +98,7 @@ async fn prometheus_metrics_exported(
         });
     }
     let (telemopts, addr, _aborter) = prom_metrics(Some(opts_builder.build().unwrap()));
-    let rt = CoreRuntime::new_assume_tokio(telemopts).unwrap();
+    let rt = CoreRuntime::new_assume_tokio(get_integ_runtime_options(telemopts)).unwrap();
     let opts = get_integ_server_options();
     let mut raw_client = opts
         .connect_no_namespace(rt.telemetry().get_temporal_metric_meter())
@@ -103,7 +107,7 @@ async fn prometheus_metrics_exported(
     assert!(raw_client.get_client().capabilities().is_some());
 
     let _ = raw_client
-        .list_namespaces(ListNamespacesRequest::default())
+        .list_namespaces(ListNamespacesRequest::default().into_request())
         .await
         .unwrap();
 
@@ -145,7 +149,7 @@ async fn prometheus_metrics_exported(
 async fn one_slot_worker_reports_available_slot() {
     let (telemopts, addr, _aborter) = prom_metrics(None);
     let tq = "one_slot_worker_tq";
-    let rt = CoreRuntime::new_assume_tokio(telemopts).unwrap();
+    let rt = CoreRuntime::new_assume_tokio(get_integ_runtime_options(telemopts)).unwrap();
 
     let worker_cfg = WorkerConfigBuilder::default()
         .namespace(NAMESPACE)
@@ -160,6 +164,7 @@ async fn one_slot_worker_reports_available_slot() {
         .max_outstanding_workflow_tasks(2_usize)
         .max_outstanding_nexus_tasks(1_usize)
         .workflow_task_poller_behavior(PollerBehavior::SimpleMaximum(2_usize))
+        .task_types(WorkerTaskTypes::all())
         .build()
         .unwrap();
 
@@ -398,7 +403,7 @@ async fn query_of_closed_workflow_doesnt_tick_terminal_metric(
     completion: workflow_command::Variant,
 ) {
     let (telemopts, addr, _aborter) = prom_metrics(None);
-    let rt = CoreRuntime::new_assume_tokio(telemopts).unwrap();
+    let rt = CoreRuntime::new_assume_tokio(get_integ_runtime_options(telemopts)).unwrap();
     let mut starter =
         CoreWfStarter::new_with_runtime("query_of_closed_workflow_doesnt_tick_terminal_metric", rt);
     // Disable cache to ensure replay happens completely
@@ -520,8 +525,11 @@ async fn query_of_closed_workflow_doesnt_tick_terminal_metric(
 
 #[test]
 fn runtime_new() {
-    let mut rt =
-        CoreRuntime::new(get_integ_telem_options(), TokioRuntimeBuilder::default()).unwrap();
+    let mut rt = CoreRuntime::new(
+        get_integ_runtime_options(get_integ_telem_options()),
+        TokioRuntimeBuilder::default(),
+    )
+    .unwrap();
     let handle = rt.tokio_handle();
     let _rt = handle.enter();
     let (telemopts, addr, _aborter) = prom_metrics(None);
@@ -535,7 +543,7 @@ fn runtime_new() {
             .unwrap();
         assert!(raw_client.get_client().capabilities().is_some());
         let _ = raw_client
-            .list_namespaces(ListNamespacesRequest::default())
+            .list_namespaces(ListNamespacesRequest::default().into_request())
             .await
             .unwrap();
         let body = get_text(format!("http://{addr}/metrics")).await;
@@ -567,7 +575,7 @@ async fn latency_metrics(
             .build()
             .unwrap(),
     ));
-    let rt = CoreRuntime::new_assume_tokio(telemopts).unwrap();
+    let rt = CoreRuntime::new_assume_tokio(get_integ_runtime_options(telemopts)).unwrap();
     let mut starter = CoreWfStarter::new_with_runtime("latency_metrics", rt);
     let worker = starter.get_worker().await;
     starter.start_wf().await;
@@ -621,7 +629,7 @@ async fn latency_metrics(
 #[tokio::test]
 async fn request_fail_codes() {
     let (telemopts, addr, _aborter) = prom_metrics(None);
-    let rt = CoreRuntime::new_assume_tokio(telemopts).unwrap();
+    let rt = CoreRuntime::new_assume_tokio(get_integ_runtime_options(telemopts)).unwrap();
     let opts = get_integ_server_options();
     let mut client = opts
         .connect(NAMESPACE, rt.telemetry().get_temporal_metric_meter())
@@ -629,9 +637,12 @@ async fn request_fail_codes() {
         .unwrap();
 
     // Describe namespace w/ invalid argument (unset namespace field)
-    WorkflowService::describe_namespace(&mut client, DescribeNamespaceRequest::default())
-        .await
-        .unwrap_err();
+    WorkflowService::describe_namespace(
+        &mut client,
+        DescribeNamespaceRequest::default().into_request(),
+    )
+    .await
+    .unwrap_err();
 
     let body = get_text(format!("http://{addr}/metrics")).await;
     let matching_line = body
@@ -664,8 +675,8 @@ async fn request_fail_codes_otel() {
     let mut telemopts = TelemetryOptionsBuilder::default();
     let exporter = Arc::new(exporter);
     telemopts.metrics(exporter as Arc<dyn CoreMeter>);
-
-    let rt = CoreRuntime::new_assume_tokio(telemopts.build().unwrap()).unwrap();
+    let rt = CoreRuntime::new_assume_tokio(get_integ_runtime_options(telemopts.build().unwrap()))
+        .unwrap();
     let opts = get_integ_server_options();
     let mut client = opts
         .connect(NAMESPACE, rt.telemetry().get_temporal_metric_meter())
@@ -674,9 +685,12 @@ async fn request_fail_codes_otel() {
 
     for _ in 0..10 {
         // Describe namespace w/ invalid argument (unset namespace field)
-        WorkflowService::describe_namespace(&mut client, DescribeNamespaceRequest::default())
-            .await
-            .unwrap_err();
+        WorkflowService::describe_namespace(
+            &mut client,
+            DescribeNamespaceRequest::default().into_request(),
+        )
+        .await
+        .unwrap_err();
 
         tokio::time::sleep(Duration::from_secs(1)).await;
     }
@@ -715,7 +729,7 @@ async fn docker_metrics_with_prometheus(
         .metric_prefix(test_uid.clone())
         .build()
         .unwrap();
-    let rt = CoreRuntime::new_assume_tokio(telemopts).unwrap();
+    let rt = CoreRuntime::new_assume_tokio(get_integ_runtime_options(telemopts)).unwrap();
     let test_name = "docker_metrics_with_prometheus";
     let mut starter = CoreWfStarter::new_with_runtime(test_name, rt);
     let worker = starter.get_worker().await;
@@ -769,7 +783,7 @@ async fn docker_metrics_with_prometheus(
 #[tokio::test]
 async fn activity_metrics() {
     let (telemopts, addr, _aborter) = prom_metrics(None);
-    let rt = CoreRuntime::new_assume_tokio(telemopts).unwrap();
+    let rt = CoreRuntime::new_assume_tokio(get_integ_runtime_options(telemopts)).unwrap();
     let wf_name = "activity_metrics";
     let mut starter = CoreWfStarter::new_with_runtime(wf_name, rt);
     starter
@@ -903,10 +917,15 @@ async fn activity_metrics() {
 #[tokio::test]
 async fn nexus_metrics() {
     let (telemopts, addr, _aborter) = prom_metrics(None);
-    let rt = CoreRuntime::new_assume_tokio(telemopts).unwrap();
+    let rt = CoreRuntime::new_assume_tokio(get_integ_runtime_options(telemopts)).unwrap();
     let wf_name = "nexus_metrics";
     let mut starter = CoreWfStarter::new_with_runtime(wf_name, rt);
-    starter.worker_config.no_remote_activities(true);
+    starter.worker_config.task_types(WorkerTaskTypes {
+        enable_workflows: true,
+        enable_local_activities: false,
+        enable_remote_activities: false,
+        enable_nexus: true,
+    });
     let task_queue = starter.get_task_queue().to_owned();
     let mut worker = starter.worker().await;
     let core_worker = starter.get_worker().await;
@@ -1080,10 +1099,12 @@ async fn nexus_metrics() {
 #[tokio::test]
 async fn evict_on_complete_does_not_count_as_forced_eviction() {
     let (telemopts, addr, _aborter) = prom_metrics(None);
-    let rt = CoreRuntime::new_assume_tokio(telemopts).unwrap();
+    let rt = CoreRuntime::new_assume_tokio(get_integ_runtime_options(telemopts)).unwrap();
     let wf_name = "evict_on_complete_does_not_count_as_forced_eviction";
     let mut starter = CoreWfStarter::new_with_runtime(wf_name, rt);
-    starter.worker_config.no_remote_activities(true);
+    starter
+        .worker_config
+        .task_types(WorkerTaskTypes::workflow_only());
     let mut worker = starter.worker().await;
 
     worker.register_wf(
@@ -1163,10 +1184,12 @@ where
 #[tokio::test]
 async fn metrics_available_from_custom_slot_supplier() {
     let (telemopts, addr, _aborter) = prom_metrics(None);
-    let rt = CoreRuntime::new_assume_tokio(telemopts).unwrap();
+    let rt = CoreRuntime::new_assume_tokio(get_integ_runtime_options(telemopts)).unwrap();
     let mut starter =
         CoreWfStarter::new_with_runtime("metrics_available_from_custom_slot_supplier", rt);
-    starter.worker_config.no_remote_activities(true);
+    starter
+        .worker_config
+        .task_types(WorkerTaskTypes::workflow_only());
     starter.worker_config.clear_max_outstanding_opts();
     let mut tb = TunerBuilder::default();
     tb.workflow_slot_supplier(Arc::new(MetricRecordingSlotSupplier::<WorkflowSlotKind> {
@@ -1313,3 +1336,154 @@ async fn prometheus_label_nonsense() {
     assert!(body.contains("some_counter{thing=\"foo\"} 2"));
     assert!(body.contains("some_counter{blerp=\"baz\"} 2"));
 }
+
+#[rstest::rstest]
+#[tokio::test]
+async fn sticky_queue_label_strategy(
+    #[values(
+        TaskQueueLabelStrategy::UseNormal,
+        TaskQueueLabelStrategy::UseNormalAndSticky
+    )]
+    strategy: TaskQueueLabelStrategy,
+) {
+    let (mut telemopts, addr, _aborter) = prom_metrics(Some(
+        PrometheusExporterOptionsBuilder::default()
+            .socket_addr(ANY_PORT.parse().unwrap())
+            .build()
+            .unwrap(),
+    ));
+    telemopts.task_queue_label_strategy = strategy;
+    let rt = CoreRuntime::new_assume_tokio(get_integ_runtime_options(telemopts)).unwrap();
+    let wf_name = format!("sticky_queue_label_strategy_{strategy:?}");
+    let mut starter = CoreWfStarter::new_with_runtime(&wf_name, rt);
+    // Enable sticky queues by setting a reasonable cache size
+    starter.worker_config.max_cached_workflows(10_usize);
+    starter
+        .worker_config
+        .task_types(WorkerTaskTypes::workflow_only());
+    let task_queue = starter.get_task_queue().to_owned();
+    let mut worker = starter.worker().await;
+
+    worker.register_wf(wf_name.clone(), |ctx: WfContext| async move {
+        ctx.timer(Duration::from_millis(1)).await;
+        Ok(().into())
+    });
+    worker
+        .submit_wf(
+            wf_name.clone(),
+            wf_name,
+            vec![],
+            WorkflowOptions {
+                enable_eager_workflow_start: false,
+                ..Default::default()
+            },
+        )
+        .await
+        .unwrap();
+    worker.run_until_done().await.unwrap();
+
+    // Give metrics time to be recorded
+    tokio::time::sleep(Duration::from_millis(200)).await;
+
+    let body = get_text(format!("http://{addr}/metrics")).await;
+
+    let poll_metrics: Vec<_> = body
+        .lines()
+        .filter(|l| {
+            l.contains("temporal_long_request")
+                && l.contains("operation=\"PollWorkflowTaskQueue\"")
+                && l.contains(&format!("namespace=\"{NAMESPACE}\""))
+        })
+        .collect();
+
+    assert!(!poll_metrics.is_empty(), "Should have poll metrics");
+
+    match strategy {
+        TaskQueueLabelStrategy::UseNormalAndSticky => {
+            // With UseNormalAndSticky, we should see sticky queue names
+            let has_sticky = poll_metrics.iter().any(|l| {
+                l.contains("task_queue=")
+                    && l.contains("WorkflowTask")
+                    && !l.contains(&format!("task_queue=\"{task_queue}\""))
+            });
+
+            assert!(
+                has_sticky,
+                "With UseNormalAndSticky, should see sticky queue names in metrics. Metrics:\n{}",
+                poll_metrics.join("\n")
+            );
+        }
+        TaskQueueLabelStrategy::UseNormal => {
+            // With UseNormal, ALL metrics should use the normal queue name
+            for l in &poll_metrics {
+                if l.contains("task_queue=") && l.contains("WorkflowTask") {
+                    assert!(
+                        l.contains(&format!("task_queue=\"{task_queue}\"")),
+                        "With UseNormal, all workflow task_queue labels should use normal name. Found: {l}",
+                    );
+                }
+            }
+        }
+        _ => unreachable!("Test only covers UseNormal and UseNormalAndSticky"),
+    }
+}
+
+#[tokio::test]
+async fn resource_based_tuner_metrics() {
+    use temporalio_sdk_core::ResourceBasedTuner;
+
+    let (telemopts, addr, _aborter) = prom_metrics(None);
+    let rt = CoreRuntime::new_assume_tokio(get_integ_runtime_options(telemopts)).unwrap();
+    let wf_name = "resource_based_tuner_metrics";
+    let mut starter = CoreWfStarter::new_with_runtime(wf_name, rt);
+    starter
+        .worker_config
+        .task_types(WorkerTaskTypes::workflow_only());
+    starter.worker_config.clear_max_outstanding_opts();
+
+    // Create a resource-based tuner with reasonable thresholds
+    let tuner = ResourceBasedTuner::new(0.8, 0.8);
+    starter.worker_config.tuner(Arc::new(tuner));
+
+    let mut worker = starter.worker().await;
+
+    worker.register_wf(wf_name.to_string(), |ctx: WfContext| async move {
+        ctx.timer(Duration::from_millis(100)).await;
+        Ok(().into())
+    });
+
+    worker
+        .submit_wf(
+            wf_name.to_owned(),
+            wf_name.to_owned(),
+            vec![],
+            WorkflowOptions::default(),
+        )
+        .await
+        .unwrap();
+
+    worker.run_until_done().await.unwrap();
+
+    // Give metrics time to be recorded (metrics are emitted every 1 second)
+    tokio::time::sleep(Duration::from_millis(1500)).await;
+
+    let body = get_text(format!("http://{addr}/metrics")).await;
+
+    // Verify that the resource-based tuner metrics are present
+    assert!(
+        body.contains("temporal_resource_slots_mem_usage"),
+        "Memory usage metric should be present"
+    );
+    assert!(
+        body.contains("temporal_resource_slots_cpu_usage"),
+        "CPU usage metric should be present"
+    );
+    assert!(
+        body.contains("temporal_resource_slots_mem_pid_output"),
+        "Memory PID output metric should be present"
+    );
+    assert!(
+        body.contains("temporal_resource_slots_cpu_pid_output"),
+        "CPU PID output metric should be present"
+    );
+}