@temporalio/core-bridge 1.11.6 → 1.11.8

package/sdk-core/core/src/retry_logic.rs

@@ -169,8 +169,8 @@ mod tests {
             maximum_attempts: 10,
             non_retryable_error_types: vec!["no retry".to_string()],
         };
-        assert!(rp
-            .should_retry(
+        assert!(
+            rp.should_retry(
                 1,
                 Some(&ApplicationFailureInfo {
                     r#type: "no retry".to_string(),
@@ -178,7 +178,8 @@ mod tests {
                     ..Default::default()
                 })
             )
-            .is_none());
+            .is_none()
+        );
     }
 
     #[test]
@@ -190,8 +191,8 @@ mod tests {
             maximum_attempts: 10,
             non_retryable_error_types: vec![],
         };
-        assert!(rp
-            .should_retry(
+        assert!(
+            rp.should_retry(
                 1,
                 Some(&ApplicationFailureInfo {
                     r#type: "".to_string(),
@@ -199,7 +200,8 @@ mod tests {
                     ..Default::default()
                 })
             )
-            .is_none());
+            .is_none()
+        );
     }
 
     #[test]

package/sdk-core/core/src/telemetry/log_export.rs

@@ -1,6 +1,6 @@
-use futures_channel::mpsc::{channel, Receiver, Sender};
+use futures_channel::mpsc::{Receiver, Sender, channel};
 use parking_lot::Mutex;
-use ringbuf::{consumer::Consumer, producer::Producer, traits::Split, HeapRb};
+use ringbuf::{HeapRb, consumer::Consumer, producer::Producer, traits::Split};
 use std::{collections::HashMap, fmt, sync::Arc, time::SystemTime};
 use temporal_sdk_core_api::telemetry::{CoreLog, CoreLogConsumer};
 use tracing_subscriber::Layer;
@@ -168,7 +168,7 @@ impl fmt::Debug for CoreLogStreamConsumer {
 
 struct JsonVisitor<'a>(&'a mut HashMap<String, serde_json::Value>);
 
-impl<'a> tracing::field::Visit for JsonVisitor<'a> {
+impl tracing::field::Visit for JsonVisitor<'_> {
     fn record_f64(&mut self, field: &tracing::field::Field, value: f64) {
         self.0
             .insert(field.name().to_string(), serde_json::json!(value));
@@ -216,7 +216,7 @@ impl<'a> tracing::field::Visit for JsonVisitor<'a> {
 #[cfg(test)]
 mod tests {
     use crate::{
-        telemetry::{construct_filter_string, CoreLogStreamConsumer},
+        telemetry::{CoreLogStreamConsumer, construct_filter_string},
         telemetry_init,
     };
     use futures_util::stream::StreamExt;

package/sdk-core/core/src/telemetry/metrics.rs

@@ -48,6 +48,11 @@ struct Instruments {
     la_exec_latency: Arc<dyn HistogramDuration>,
     la_exec_succeeded_latency: Arc<dyn HistogramDuration>,
     la_total: Arc<dyn Counter>,
+    nexus_poll_no_task: Arc<dyn Counter>,
+    nexus_task_schedule_to_start_latency: Arc<dyn HistogramDuration>,
+    nexus_task_e2e_latency: Arc<dyn HistogramDuration>,
+    nexus_task_execution_latency: Arc<dyn HistogramDuration>,
+    nexus_task_execution_failed: Arc<dyn Counter>,
     worker_registered: Arc<dyn Counter>,
     num_pollers: Arc<dyn Gauge>,
     task_slots_available: Arc<dyn Gauge>,
@@ -55,7 +60,7 @@ struct Instruments {
     sticky_cache_hit: Arc<dyn Counter>,
     sticky_cache_miss: Arc<dyn Counter>,
     sticky_cache_size: Arc<dyn Gauge>,
-    sticky_cache_evictions: Arc<dyn Counter>,
+    sticky_cache_forced_evictions: Arc<dyn Counter>,
 }
 
 impl MetricsContext {
@@ -225,6 +230,39 @@ impl MetricsContext {
         self.instruments.la_total.add(1, &self.kvs);
     }
 
+    /// A nexus long poll timed out
+    pub(crate) fn nexus_poll_timeout(&self) {
+        self.instruments.nexus_poll_no_task.add(1, &self.kvs);
+    }
+
+    /// Record nexus task schedule to start time
+    pub(crate) fn nexus_task_sched_to_start_latency(&self, dur: Duration) {
+        self.instruments
+            .nexus_task_schedule_to_start_latency
+            .record(dur, &self.kvs);
+    }
+
+    /// Record nexus task end-to-end time
+    pub(crate) fn nexus_task_e2e_latency(&self, dur: Duration) {
+        self.instruments
+            .nexus_task_e2e_latency
+            .record(dur, &self.kvs);
+    }
+
+    /// Record nexus task execution time
+    pub(crate) fn nexus_task_execution_latency(&self, dur: Duration) {
+        self.instruments
+            .nexus_task_execution_latency
+            .record(dur, &self.kvs);
+    }
+
+    /// Record a nexus task execution failure
+    pub(crate) fn nexus_task_execution_failed(&self) {
+        self.instruments
+            .nexus_task_execution_failed
+            .add(1, &self.kvs);
+    }
+
     /// A worker was registered
     pub(crate) fn worker_registered(&self) {
         self.instruments.worker_registered.add(1, &self.kvs);
@@ -263,8 +301,10 @@ impl MetricsContext {
     }
 
     /// Count a workflow being evicted from the cache
-    pub(crate) fn cache_eviction(&self) {
-        self.instruments.sticky_cache_evictions.add(1, &self.kvs);
+    pub(crate) fn forced_cache_eviction(&self) {
+        self.instruments
+            .sticky_cache_forced_evictions
+            .add(1, &self.kvs);
     }
 }
 
@@ -292,7 +332,7 @@ impl Instruments {
                 unit: "".into(),
             }),
             wf_e2e_latency: meter.histogram_duration(MetricParameters {
-                name: WF_E2E_LATENCY_NAME.into(),
+                name: WORKFLOW_E2E_LATENCY_HISTOGRAM_NAME.into(),
                 unit: "duration".into(),
                 description: "Histogram of total workflow execution latencies".into(),
             }),
@@ -312,17 +352,17 @@ impl Instruments {
                 unit: "".into(),
             }),
             wf_task_sched_to_start_latency: meter.histogram_duration(MetricParameters {
-                name: WF_TASK_SCHED_TO_START_LATENCY_NAME.into(),
+                name: WORKFLOW_TASK_SCHED_TO_START_LATENCY_HISTOGRAM_NAME.into(),
                 unit: "duration".into(),
                 description: "Histogram of workflow task schedule-to-start latencies".into(),
             }),
             wf_task_replay_latency: meter.histogram_duration(MetricParameters {
-                name: WF_TASK_REPLAY_LATENCY_NAME.into(),
+                name: WORKFLOW_TASK_REPLAY_LATENCY_HISTOGRAM_NAME.into(),
                 unit: "duration".into(),
                 description: "Histogram of workflow task replay latencies".into(),
             }),
             wf_task_execution_latency: meter.histogram_duration(MetricParameters {
-                name: WF_TASK_EXECUTION_LATENCY_NAME.into(),
+                name: WORKFLOW_TASK_EXECUTION_LATENCY_HISTOGRAM_NAME.into(),
                 unit: "duration".into(),
                 description: "Histogram of workflow task execution (not replay) latencies".into(),
             }),
@@ -342,12 +382,12 @@ impl Instruments {
                 unit: "".into(),
             }),
             act_sched_to_start_latency: meter.histogram_duration(MetricParameters {
-                name: ACT_SCHED_TO_START_LATENCY_NAME.into(),
+                name: ACTIVITY_SCHED_TO_START_LATENCY_HISTOGRAM_NAME.into(),
                 unit: "duration".into(),
                 description: "Histogram of activity schedule-to-start latencies".into(),
             }),
             act_exec_latency: meter.histogram_duration(MetricParameters {
-                name: ACT_EXEC_LATENCY_NAME.into(),
+                name: ACTIVITY_EXEC_LATENCY_HISTOGRAM_NAME.into(),
                 unit: "duration".into(),
                 description: "Histogram of activity execution latencies".into(),
             }),
@@ -384,6 +424,31 @@ impl Instruments {
                 description: "Count of local activities executed".into(),
                 unit: "".into(),
             }),
+            nexus_poll_no_task: meter.counter(MetricParameters {
+                name: "nexus_poll_no_task".into(),
+                description: "Count of nexus task queue poll timeouts (no new task)".into(),
+                unit: "".into(),
+            }),
+            nexus_task_schedule_to_start_latency: meter.histogram_duration(MetricParameters {
+                name: "nexus_task_schedule_to_start_latency".into(),
+                unit: "duration".into(),
+                description: "Histogram of nexus task schedule-to-start latencies".into(),
+            }),
+            nexus_task_e2e_latency: meter.histogram_duration(MetricParameters {
+                name: "nexus_task_endtoend_latency".into(),
+                unit: "duration".into(),
+                description: "Histogram of nexus task end-to-end latencies".into(),
+            }),
+            nexus_task_execution_latency: meter.histogram_duration(MetricParameters {
+                name: "nexus_task_execution_latency".into(),
+                unit: "duration".into(),
+                description: "Histogram of nexus task execution latencies".into(),
+            }),
+            nexus_task_execution_failed: meter.counter(MetricParameters {
+                name: "nexus_task_execution_failed".into(),
+                description: "Count of nexus task execution failures".into(),
+                unit: "".into(),
+            }),
             // name kept as worker start for compat with old sdk / what users expect
             worker_registered: meter.counter(MetricParameters {
                 name: "worker_start".into(),
@@ -423,7 +488,7 @@ impl Instruments {
                 description: "Current number of cached workflows".into(),
                 unit: "".into(),
             }),
-            sticky_cache_evictions: meter.counter(MetricParameters {
+            sticky_cache_forced_evictions: meter.counter(MetricParameters {
                 name: "sticky_cache_total_forced_eviction".into(),
                 description: "Count of evictions of cached workflows".into(),
                 unit: "".into(),
@@ -450,6 +515,9 @@ pub(crate) fn workflow_sticky_poller() -> MetricKeyValue {
 pub(crate) fn activity_poller() -> MetricKeyValue {
     MetricKeyValue::new(KEY_POLLER_TYPE, "activity_task")
 }
+pub(crate) fn nexus_poller() -> MetricKeyValue {
+    MetricKeyValue::new(KEY_POLLER_TYPE, "nexus_task")
+}
 pub(crate) fn task_queue(tq: String) -> MetricKeyValue {
     MetricKeyValue::new(KEY_TASK_QUEUE, tq)
 }
@@ -468,18 +536,27 @@ pub(crate) fn activity_worker_type() -> MetricKeyValue {
 pub(crate) fn local_activity_worker_type() -> MetricKeyValue {
     MetricKeyValue::new(KEY_WORKER_TYPE, "LocalActivityWorker")
 }
+pub(crate) fn nexus_worker_type() -> MetricKeyValue {
+    MetricKeyValue::new(KEY_WORKER_TYPE, "NexusWorker")
+}
 pub(crate) fn eager(is_eager: bool) -> MetricKeyValue {
     MetricKeyValue::new(KEY_EAGER, is_eager)
 }
 pub(crate) enum FailureReason {
     Nondeterminism,
     Workflow,
+    Timeout,
+    NexusOperation(String),
+    NexusHandlerError(String),
 }
 impl Display for FailureReason {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         let str = match self {
-            FailureReason::Nondeterminism => "NonDeterminismError",
-            FailureReason::Workflow => "WorkflowError",
+            FailureReason::Nondeterminism => "NonDeterminismError".to_owned(),
+            FailureReason::Workflow => "WorkflowError".to_owned(),
+            FailureReason::Timeout => "timeout".to_owned(),
+            FailureReason::NexusOperation(op) => format!("operation_{}", op),
+            FailureReason::NexusHandlerError(op) => format!("handler_error_{}", op),
         };
         write!(f, "{}", str)
     }
@@ -496,13 +573,20 @@ pub(crate) fn failure_reason(reason: FailureReason) -> MetricKeyValue {
     MetricKeyValue::new(KEY_TASK_FAILURE_TYPE, reason.to_string())
 }
 
-pub(super) const WF_E2E_LATENCY_NAME: &str = "workflow_endtoend_latency";
-pub(super) const WF_TASK_SCHED_TO_START_LATENCY_NAME: &str =
+/// The string name (which may be prefixed) for this metric
+pub const WORKFLOW_E2E_LATENCY_HISTOGRAM_NAME: &str = "workflow_endtoend_latency";
+/// The string name (which may be prefixed) for this metric
+pub const WORKFLOW_TASK_SCHED_TO_START_LATENCY_HISTOGRAM_NAME: &str =
     "workflow_task_schedule_to_start_latency";
-pub(super) const WF_TASK_REPLAY_LATENCY_NAME: &str = "workflow_task_replay_latency";
-pub(super) const WF_TASK_EXECUTION_LATENCY_NAME: &str = "workflow_task_execution_latency";
-pub(super) const ACT_SCHED_TO_START_LATENCY_NAME: &str = "activity_schedule_to_start_latency";
-pub(super) const ACT_EXEC_LATENCY_NAME: &str = "activity_execution_latency";
+/// The string name (which may be prefixed) for this metric
+pub const WORKFLOW_TASK_REPLAY_LATENCY_HISTOGRAM_NAME: &str = "workflow_task_replay_latency";
+/// The string name (which may be prefixed) for this metric
+pub const WORKFLOW_TASK_EXECUTION_LATENCY_HISTOGRAM_NAME: &str = "workflow_task_execution_latency";
+/// The string name (which may be prefixed) for this metric
+pub const ACTIVITY_SCHED_TO_START_LATENCY_HISTOGRAM_NAME: &str =
+    "activity_schedule_to_start_latency";
+/// The string name (which may be prefixed) for this metric
+pub const ACTIVITY_EXEC_LATENCY_HISTOGRAM_NAME: &str = "activity_execution_latency";
 pub(super) const NUM_POLLERS_NAME: &str = "num_pollers";
 pub(super) const TASK_SLOTS_AVAILABLE_NAME: &str = "worker_task_slots_available";
 pub(super) const TASK_SLOTS_USED_NAME: &str = "worker_task_slots_used";
@@ -533,7 +617,7 @@ macro_rules! define_latency_buckets {
 
 define_latency_buckets!(
     (
-        WF_E2E_LATENCY_NAME,
+        WORKFLOW_E2E_LATENCY_HISTOGRAM_NAME,
         WF_LATENCY_MS_BUCKETS,
         WF_LATENCY_S_BUCKETS,
         [
@@ -556,19 +640,21 @@ define_latency_buckets!(
         ]
     ),
     (
-        WF_TASK_EXECUTION_LATENCY_NAME | WF_TASK_REPLAY_LATENCY_NAME,
+        WORKFLOW_TASK_EXECUTION_LATENCY_HISTOGRAM_NAME
+            | WORKFLOW_TASK_REPLAY_LATENCY_HISTOGRAM_NAME,
         WF_TASK_MS_BUCKETS,
         WF_TASK_S_BUCKETS,
         [1., 10., 20., 50., 100., 200., 500., 1000.]
     ),
     (
-        ACT_EXEC_LATENCY_NAME,
+        ACTIVITY_EXEC_LATENCY_HISTOGRAM_NAME,
         ACT_EXE_MS_BUCKETS,
         ACT_EXE_S_BUCKETS,
         [50., 100., 500., 1000., 5000., 10_000., 60_000.]
     ),
     (
-        WF_TASK_SCHED_TO_START_LATENCY_NAME | ACT_SCHED_TO_START_LATENCY_NAME,
+        WORKFLOW_TASK_SCHED_TO_START_LATENCY_HISTOGRAM_NAME
+            | ACTIVITY_SCHED_TO_START_LATENCY_HISTOGRAM_NAME,
         TASK_SCHED_TO_START_MS_BUCKETS,
         TASK_SCHED_TO_START_S_BUCKETS,
         [100., 500., 1000., 5000., 10_000., 100_000., 1_000_000.]
@@ -818,8 +904,8 @@ mod tests {
     use super::*;
     use std::any::Any;
     use temporal_sdk_core_api::telemetry::{
-        metrics::{BufferInstrumentRef, CustomMetricAttributes},
         METRIC_PREFIX,
+        metrics::{BufferInstrumentRef, CustomMetricAttributes},
     };
     use tracing::subscriber::NoSubscriber;
 
@@ -858,7 +944,7 @@ mod tests {
             true,
         );
         let mc = MetricsContext::top_level("foo".to_string(), "q".to_string(), &telem_instance);
-        mc.cache_eviction();
+        mc.forced_cache_eviction();
         let events = call_buffer.retrieve();
         let a1 = assert_matches!(
             &events[0],
@@ -875,7 +961,7 @@ mod tests {
         a1.set(Arc::new(DummyCustomAttrs(1))).unwrap();
         // Verify all metrics are created. This number will need to get updated any time a metric
         // is added.
-        let num_metrics = 30;
+        let num_metrics = 35;
         #[allow(clippy::needless_range_loop)] // Sorry clippy, this reads easier.
         for metric_num in 1..=num_metrics {
             let hole = assert_matches!(&events[metric_num],

package/sdk-core/core/src/telemetry/mod.rs

@@ -9,7 +9,12 @@ mod otel;
 mod prometheus_server;
 
 #[cfg(feature = "otel")]
-pub use metrics::{default_buckets_for, MetricsCallBuffer};
+pub use metrics::{
+    ACTIVITY_EXEC_LATENCY_HISTOGRAM_NAME, ACTIVITY_SCHED_TO_START_LATENCY_HISTOGRAM_NAME,
+    MetricsCallBuffer, WORKFLOW_E2E_LATENCY_HISTOGRAM_NAME,
+    WORKFLOW_TASK_EXECUTION_LATENCY_HISTOGRAM_NAME, WORKFLOW_TASK_REPLAY_LATENCY_HISTOGRAM_NAME,
+    WORKFLOW_TASK_SCHED_TO_START_LATENCY_HISTOGRAM_NAME, default_buckets_for,
+};
 #[cfg(feature = "otel")]
 pub use otel::{build_otlp_metric_exporter, start_prometheus_metric_exporter};
 
@@ -23,16 +28,16 @@ use std::{
     collections::VecDeque,
     env,
     sync::{
-        atomic::{AtomicBool, Ordering},
         Arc,
+        atomic::{AtomicBool, Ordering},
     },
 };
 use temporal_sdk_core_api::telemetry::{
-    metrics::{CoreMeter, MetricKeyValue, NewAttributes, TemporalMeter},
     CoreLog, CoreTelemetry, Logger, TelemetryOptions,
+    metrics::{CoreMeter, MetricKeyValue, NewAttributes, TemporalMeter},
 };
 use tracing::{Level, Subscriber};
-use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Layer};
+use tracing_subscriber::{EnvFilter, Layer, layer::SubscriberExt};
 
 const TELEM_SERVICE_NAME: &str = "temporal-core-sdk";
 
@@ -271,11 +276,13 @@ where
     }
 }
 
+/// Helpers for test initialization
 #[cfg(test)]
 pub mod test_initters {
     use super::*;
     use temporal_sdk_core_api::telemetry::TelemetryOptionsBuilder;
 
+    /// Turn on logging to the console
     #[allow(dead_code)] // Not always used, called to enable for debugging when needed
     pub fn test_telem_console() {
         telemetry_init_global(